Skip to content

Commit bea65bd

Browse files
committed
src, scripts: better directory cache
Changes the way the KV directory cache works: `scripts/handle-mainline-release.mjs` is triggered by the update links action. Updates the cache entries for the directories that have symlinks. Doesn't add the cache entry for the new released version anymore. When KV listings are enabled, we try fetching the directory from the KV directory cache like usual. If the directory isn't found however, then we fall back to R2. If R2 has the directory, then we return that result and then cache the R2 result in the KV cache for however long `CACHE_KV_TTL` is (currently an arbitary 6 hours). Fixes #1034 Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com>
1 parent 27a6ea3 commit bea65bd

12 files changed

Lines changed: 374 additions & 230 deletions

File tree

‎.env.example‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,30 @@
11
# Your Cloudflare account tag.
2-
#
2+
#
33
# Needed for:
44
# - Directory cache scripts
55
CLOUDFLARE_ACCOUNT_ID=
66

77
# Cloudflare V4 API token.
8-
#
8+
#
99
# Needed for:
1010
# - Directory cache scripts
11-
#
11+
#
1212
# Required permissions:
1313
# - `Workers KV Storage`: Edit
1414
# - `Workers R2 Storage`: Read
15-
#
15+
#
1616
# See https://developers.cloudflare.com/fundamentals/api/get-started/create-token/
1717
CLOUDFLARE_API_TOKEN=
1818

1919
# S3 credentials for your R2 bucket.
20-
#
20+
#
2121
# Needed for:
2222
# - Directory listings in the worker.
2323
# - Directory cache scripts
24-
#
24+
#
2525
# Required permissions:
2626
# - `Object Read Only`
27-
#
27+
#
2828
# See https://dash.cloudflare.com/?account=/r2/api-tokens
2929
S3_ACCESS_KEY_ID=
3030
S3_ACCESS_KEY_SECRET=

‎.github/workflows/build-directory-cache.yaml‎

Lines changed: 0 additions & 82 deletions
This file was deleted.

‎.github/workflows/update-links.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,11 @@ on:
77
# Triggered by https://github.com/nodejs/node/blob/main/.github/workflows/update-release-links.yml
88
workflow_dispatch:
99
inputs:
10+
# Not used but kept around to not break Node core ci since it passes this
1011
version:
1112
description: 'Node.js version (ex/ `v20.0.0`)'
12-
required: true
13+
required: false
1314
type: string
14-
schedule:
15-
- cron: '0 0 * * *'
1615

1716
concurrency:
1817
group: update-redirect-links
@@ -60,7 +59,7 @@ jobs:
6059
run: npm ci && npm update nodejs-latest-linker --save
6160

6261
- name: Update Directory Cache
63-
run: node scripts/update-directory-cache.mjs "$VERSION_INPUT" && node --run format
62+
run: node scripts/handle-mainline-release.mjs && node --run format
6463
env:
6564
VERSION_INPUT: "${{ inputs.version || '' }}"
6665
CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}

‎e2e-tests/directory.kv.test.ts‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { createExecutionContext } from 'cloudflare:test';
22
import { env } from 'cloudflare:workers';
3-
import { test, beforeAll, expect, vi } from 'vitest';
3+
import { test, beforeAll, expect, vi, afterEach, MockInstance, beforeEach } from 'vitest';
44
import {
55
populateDirectoryCacheWithDevBucket,
66
populateR2WithDevBucket,
@@ -14,6 +14,10 @@ const mockedEnv: Env = {
1414
ENVIRONMENT: 'e2e-tests',
1515
LOG_ERRORS: true,
1616
USE_KV: true,
17+
S3_ENDPOINT: 'https://s3.mock',
18+
S3_ACCESS_KEY_ID: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
19+
S3_ACCESS_KEY_SECRET:
20+
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
1721
};
1822

1923
vi.mock(
@@ -38,11 +42,68 @@ vi.mock(
3842
}
3943
);
4044

45+
const s3Url = new URL(mockedEnv.S3_ENDPOINT);
46+
s3Url.host = `${mockedEnv.BUCKET_NAME}.${s3Url.host}`;
47+
48+
const S3_DIRECTORY_RESULT = `
49+
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
50+
<Name>dist-prod</Name>
51+
<Prefix />
52+
<Marker />
53+
<MaxKeys>1000</MaxKeys>
54+
<IsTruncated>false</IsTruncated>
55+
<CommonPrefixes>
56+
<Prefix>nodejs/release/v1.0.0/latest/</Prefix>
57+
</CommonPrefixes>
58+
<Contents>
59+
<ETag>"asd123"</ETag>
60+
<Key>nodejs/release/v1.0.0/index.json</Key>
61+
<LastModified>2023-09-12T05:43:00.000Z</LastModified>
62+
<Size>18</Size>
63+
</Contents>
64+
</ListBucketResult>`;
65+
66+
const S3_EMPTY_DIRECTORY_RESULT = `
67+
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
68+
<Name>dist-prod</Name>
69+
<Prefix />
70+
<Marker />
71+
<MaxKeys>1000</MaxKeys>
72+
<IsTruncated>false</IsTruncated>
73+
</ListBucketResult>`;
74+
75+
let fetchSpy: MockInstance<typeof fetch>;
76+
4177
beforeAll(async () => {
4278
await populateR2WithDevBucket();
4379
await populateDirectoryCacheWithDevBucket();
4480
});
4581

82+
beforeEach(() => {
83+
fetchSpy = vi
84+
.spyOn(globalThis, 'fetch')
85+
.mockRejectedValue(new Error('Unexpected outbound request'));
86+
})
87+
88+
afterEach(() => {
89+
vi.restoreAllMocks();
90+
});
91+
92+
function mockS3Response(body: string, prefix?: string): void {
93+
fetchSpy.mockImplementationOnce(async (input, init) => {
94+
const url = new URL(new Request(input, init).url);
95+
96+
if (
97+
url.origin !== s3Url.origin ||
98+
(prefix !== undefined && url.searchParams.get('prefix') !== prefix)
99+
) {
100+
throw new Error(`Unexpected request to ${url}`);
101+
}
102+
103+
return new Response(body);
104+
});
105+
}
106+
46107
// Ensure essential endpoints are routable
47108
for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
48109
test(`GET \`${path}\` returns 200`, async () => {
@@ -62,6 +123,11 @@ for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
62123
}
63124

64125
test('GET `/dist/unknown-directory/` returns 404', async () => {
126+
mockS3Response(
127+
S3_EMPTY_DIRECTORY_RESULT,
128+
'nodejs/release/unknown-directory/'
129+
);
130+
65131
const ctx = createExecutionContext();
66132

67133
const res = await worker.fetch(
@@ -75,6 +141,23 @@ test('GET `/dist/unknown-directory/` returns 404', async () => {
75141
expect(await res.text()).toStrictEqual('Directory not found');
76142
});
77143

144+
test('GET `/dist/missing-in-kv-but-in-r2/` returns 200', async () => {
145+
mockS3Response(S3_DIRECTORY_RESULT);
146+
147+
const ctx = createExecutionContext();
148+
149+
const res = await worker.fetch(
150+
new Request(`https://localhost/dist/missing-in-kv-but-in-r2/`),
151+
mockedEnv,
152+
ctx
153+
);
154+
155+
// Consume body promise
156+
await res.text();
157+
158+
expect(res.status).toBe(200);
159+
});
160+
78161
test('GET `/dist` redirects to `/dist/`', async () => {
79162
const ctx = createExecutionContext();
80163

‎scripts/constants.mjs‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,4 @@ export const STATIC_FILE_SYMLINKS_PATH = join(
6161
* Paths in the R2 bucket that we should always be updating whenever a new
6262
* version releases.
6363
*/
64-
export const ALWAYS_UPDATED_PATHS = [
65-
RELEASE_DIR,
66-
DOCS_DIR,
67-
NIGHTLY_RELEASES_DIR,
68-
RC_RELEASES_DIR,
69-
TEST_RELEASES_DIR,
70-
V8_CANARY_RELEASES_DIR,
71-
];
64+
export const ALWAYS_UPDATED_PATHS = [RELEASE_DIR, DOCS_DIR];
Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
#!/usr/bin/env node
22

33
/**
4-
* Update the most commonly updated paths in the directory cache for a Node.js
5-
* release.
4+
* Updates the cached directory listings for paths that have symlinks for the
5+
* mainline release bucket.
66
*
7-
* Usage: ./update-directory-cache.mjs [version]
8-
* Example: ./update-directory-cache.mjs v24.0.0
7+
* Usage: ./handle-mainline-release.mjs
98
*
109
* If no version is provided, this just updates what's in {@link ALWAYS_UPDATED_PATHS}
1110
*/
@@ -22,8 +21,6 @@ import { getLatestVersionMapping } from './utils/getLatestVersionMapping.mjs';
2221
import { createCloudflareClient, writeKeysToKv } from './utils/kv.mjs';
2322
import { createS3Client, listR2DirectoryRecursive } from './utils/r2.mjs';
2423

25-
const VERSION = process.argv[2]?.toLowerCase();
26-
2724
// Ensure all necessary environment variables are set
2825
for (const envVar of [
2926
'CLOUDFLARE_API_TOKEN',
@@ -67,26 +64,6 @@ await Promise.all(
6764
)
6865
);
6966

70-
if (VERSION !== undefined && VERSION !== '') {
71-
if (!VERSION.startsWith('v')) {
72-
throw new TypeError(
73-
'provided version not in correct format, expected vX.X.X'
74-
);
75-
}
76-
77-
console.log(`Listing version ${VERSION}`);
78-
79-
const versionDirectory = await listR2DirectoryRecursive(
80-
s3Client,
81-
PROD_BUCKET,
82-
`${RELEASE_DIR}${VERSION}/`
83-
);
84-
85-
for (const [k, v] of versionDirectory.entries()) {
86-
directories.set(k, v);
87-
}
88-
}
89-
9067
const latestVersions = await getLatestVersionMapping(
9168
s3Client,
9269
directories.get(RELEASE_DIR),
@@ -95,6 +72,8 @@ const latestVersions = await getLatestVersionMapping(
9572

9673
await addSymlinksToDirectoryCache(s3Client, directories, latestVersions);
9774

98-
// Update keys in the directory cache
75+
// Update keys in the directory cache. These intentionally do not have an
76+
// expiration set since they were made with info that the worker doesn't have
77+
// when deployed.
9978
console.log(`Writing ${directories.size} keys to directory cache...`);
10079
await writeKeysToKv(cfClient, DIRECTORY_CACHE_NAMESPACE_ID, directories);

‎scripts/utils/addSymlinksToDirectoryCache.mjs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,10 +124,10 @@ export async function addStaticFileSymlinksToCache(
124124
*/
125125
const fileSymlinks = JSON.parse(fileSymlinksFile);
126126

127-
// This file is an exception for two reasons:
128-
// 1. It's dynamic and points to the latest Node version
129-
// 2. It's handled specially and is technically in the root directory,
130-
// which we should never be listing thus don't need to add it to the
127+
// These files are an exception for two reasons:
128+
// 1. They're dynamic and point to the latest Node version
129+
// 2. They're handled specially and are technically in the root directory,
130+
// which we should never be listing thus don't need to add them to the
131131
// directory cache.
132132
// So, let's remove them then add them back once we're done handling the rest
133133
// of the static file symlinks.

‎src/constants/cache.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,8 @@ export const CACHE_HEADERS = {
44
mutable: 'public, max-age=3600, s-maxage=14400',
55
failure: 'private, no-cache, no-store, max-age=0, must-revalidate',
66
};
7+
8+
/**
9+
* Ttl for elements in the KV cache in seconds. This is completely arbitrary
10+
*/
11+
export const KV_CACHE_TTL = 60 * 60 * 6;

0 commit comments

Comments
 (0)