Skip to content

Commit c4f8a76

Browse files
committed
src, scripts: better directory cache
Changes the way the KV directory cache works: `scripts/handle-mainline-release.mjs` is triggered by the update links action. Updates the cache entries for the directories that have symlinks. Doesn't add the cache entry for the new released version anymore. When KV listings are enabled, we try fetching the directory from the KV directory cache like usual. If the directory isn't found however, then we fall back to R2. If R2 has the directory, then we return that result and then cache the R2 result in the KV cache for however long `CACHE_KV_TTL` is (currently an arbitary 6 hours). Fixes #1034 Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com>
1 parent 27a6ea3 commit c4f8a76

12 files changed

Lines changed: 382 additions & 230 deletions

File tree

‎.env.example‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,30 @@
11
# Your Cloudflare account tag.
2-
#
2+
#
33
# Needed for:
44
# - Directory cache scripts
55
CLOUDFLARE_ACCOUNT_ID=
66

77
# Cloudflare V4 API token.
8-
#
8+
#
99
# Needed for:
1010
# - Directory cache scripts
11-
#
11+
#
1212
# Required permissions:
1313
# - `Workers KV Storage`: Edit
1414
# - `Workers R2 Storage`: Read
15-
#
15+
#
1616
# See https://developers.cloudflare.com/fundamentals/api/get-started/create-token/
1717
CLOUDFLARE_API_TOKEN=
1818

1919
# S3 credentials for your R2 bucket.
20-
#
20+
#
2121
# Needed for:
2222
# - Directory listings in the worker.
2323
# - Directory cache scripts
24-
#
24+
#
2525
# Required permissions:
2626
# - `Object Read Only`
27-
#
27+
#
2828
# See https://dash.cloudflare.com/?account=/r2/api-tokens
2929
S3_ACCESS_KEY_ID=
3030
S3_ACCESS_KEY_SECRET=

‎.github/workflows/build-directory-cache.yaml‎

Lines changed: 0 additions & 82 deletions
This file was deleted.

‎.github/workflows/update-links.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,11 @@ on:
77
# Triggered by https://github.com/nodejs/node/blob/main/.github/workflows/update-release-links.yml
88
workflow_dispatch:
99
inputs:
10+
# Not used but kept around to not break Node core ci since it passes this
1011
version:
1112
description: 'Node.js version (ex/ `v20.0.0`)'
12-
required: true
13+
required: false
1314
type: string
14-
schedule:
15-
- cron: '0 0 * * *'
1615

1716
concurrency:
1817
group: update-redirect-links
@@ -60,7 +59,7 @@ jobs:
6059
run: npm ci && npm update nodejs-latest-linker --save
6160

6261
- name: Update Directory Cache
63-
run: node scripts/update-directory-cache.mjs "$VERSION_INPUT" && node --run format
62+
run: node scripts/handle-mainline-release.mjs && node --run format
6463
env:
6564
VERSION_INPUT: "${{ inputs.version || '' }}"
6665
CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}

‎e2e-tests/directory.kv.test.ts‎

Lines changed: 92 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
import { createExecutionContext } from 'cloudflare:test';
22
import { env } from 'cloudflare:workers';
3-
import { test, beforeAll, expect, vi } from 'vitest';
3+
import {
4+
test,
5+
beforeAll,
6+
expect,
7+
vi,
8+
afterEach,
9+
MockInstance,
10+
beforeEach,
11+
} from 'vitest';
412
import {
513
populateDirectoryCacheWithDevBucket,
614
populateR2WithDevBucket,
@@ -14,6 +22,10 @@ const mockedEnv: Env = {
1422
ENVIRONMENT: 'e2e-tests',
1523
LOG_ERRORS: true,
1624
USE_KV: true,
25+
S3_ENDPOINT: 'https://s3.mock',
26+
S3_ACCESS_KEY_ID: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
27+
S3_ACCESS_KEY_SECRET:
28+
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
1729
};
1830

1931
vi.mock(
@@ -38,11 +50,68 @@ vi.mock(
3850
}
3951
);
4052

53+
const s3Url = new URL(mockedEnv.S3_ENDPOINT);
54+
s3Url.host = `${mockedEnv.BUCKET_NAME}.${s3Url.host}`;
55+
56+
const S3_DIRECTORY_RESULT = `
57+
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
58+
<Name>dist-prod</Name>
59+
<Prefix />
60+
<Marker />
61+
<MaxKeys>1000</MaxKeys>
62+
<IsTruncated>false</IsTruncated>
63+
<CommonPrefixes>
64+
<Prefix>nodejs/release/v1.0.0/latest/</Prefix>
65+
</CommonPrefixes>
66+
<Contents>
67+
<ETag>"asd123"</ETag>
68+
<Key>nodejs/release/v1.0.0/index.json</Key>
69+
<LastModified>2023-09-12T05:43:00.000Z</LastModified>
70+
<Size>18</Size>
71+
</Contents>
72+
</ListBucketResult>`;
73+
74+
const S3_EMPTY_DIRECTORY_RESULT = `
75+
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
76+
<Name>dist-prod</Name>
77+
<Prefix />
78+
<Marker />
79+
<MaxKeys>1000</MaxKeys>
80+
<IsTruncated>false</IsTruncated>
81+
</ListBucketResult>`;
82+
83+
let fetchSpy: MockInstance<typeof fetch>;
84+
4185
beforeAll(async () => {
4286
await populateR2WithDevBucket();
4387
await populateDirectoryCacheWithDevBucket();
4488
});
4589

90+
beforeEach(() => {
91+
fetchSpy = vi
92+
.spyOn(globalThis, 'fetch')
93+
.mockRejectedValue(new Error('Unexpected outbound request'));
94+
});
95+
96+
afterEach(() => {
97+
vi.restoreAllMocks();
98+
});
99+
100+
function mockS3Response(body: string, prefix?: string): void {
101+
fetchSpy.mockImplementationOnce(async (input, init) => {
102+
const url = new URL(new Request(input, init).url);
103+
104+
if (
105+
url.origin !== s3Url.origin ||
106+
(prefix !== undefined && url.searchParams.get('prefix') !== prefix)
107+
) {
108+
throw new Error(`Unexpected request to ${url}`);
109+
}
110+
111+
return new Response(body);
112+
});
113+
}
114+
46115
// Ensure essential endpoints are routable
47116
for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
48117
test(`GET \`${path}\` returns 200`, async () => {
@@ -62,6 +131,11 @@ for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
62131
}
63132

64133
test('GET `/dist/unknown-directory/` returns 404', async () => {
134+
mockS3Response(
135+
S3_EMPTY_DIRECTORY_RESULT,
136+
'nodejs/release/unknown-directory/'
137+
);
138+
65139
const ctx = createExecutionContext();
66140

67141
const res = await worker.fetch(
@@ -75,6 +149,23 @@ test('GET `/dist/unknown-directory/` returns 404', async () => {
75149
expect(await res.text()).toStrictEqual('Directory not found');
76150
});
77151

152+
test('GET `/dist/missing-in-kv-but-in-r2/` returns 200', async () => {
153+
mockS3Response(S3_DIRECTORY_RESULT);
154+
155+
const ctx = createExecutionContext();
156+
157+
const res = await worker.fetch(
158+
new Request(`https://localhost/dist/missing-in-kv-but-in-r2/`),
159+
mockedEnv,
160+
ctx
161+
);
162+
163+
// Consume body promise
164+
await res.text();
165+
166+
expect(res.status).toBe(200);
167+
});
168+
78169
test('GET `/dist` redirects to `/dist/`', async () => {
79170
const ctx = createExecutionContext();
80171

‎scripts/constants.mjs‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,4 @@ export const STATIC_FILE_SYMLINKS_PATH = join(
6161
* Paths in the R2 bucket that we should always be updating whenever a new
6262
* version releases.
6363
*/
64-
export const ALWAYS_UPDATED_PATHS = [
65-
RELEASE_DIR,
66-
DOCS_DIR,
67-
NIGHTLY_RELEASES_DIR,
68-
RC_RELEASES_DIR,
69-
TEST_RELEASES_DIR,
70-
V8_CANARY_RELEASES_DIR,
71-
];
64+
export const ALWAYS_UPDATED_PATHS = [RELEASE_DIR, DOCS_DIR];
Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
#!/usr/bin/env node
22

33
/**
4-
* Update the most commonly updated paths in the directory cache for a Node.js
5-
* release.
4+
* Updates the cached directory listings for paths that have symlinks for the
5+
* mainline release bucket.
66
*
7-
* Usage: ./update-directory-cache.mjs [version]
8-
* Example: ./update-directory-cache.mjs v24.0.0
7+
* Usage: ./handle-mainline-release.mjs
98
*
109
* If no version is provided, this just updates what's in {@link ALWAYS_UPDATED_PATHS}
1110
*/
@@ -22,8 +21,6 @@ import { getLatestVersionMapping } from './utils/getLatestVersionMapping.mjs';
2221
import { createCloudflareClient, writeKeysToKv } from './utils/kv.mjs';
2322
import { createS3Client, listR2DirectoryRecursive } from './utils/r2.mjs';
2423

25-
const VERSION = process.argv[2]?.toLowerCase();
26-
2724
// Ensure all necessary environment variables are set
2825
for (const envVar of [
2926
'CLOUDFLARE_API_TOKEN',
@@ -67,26 +64,6 @@ await Promise.all(
6764
)
6865
);
6966

70-
if (VERSION !== undefined && VERSION !== '') {
71-
if (!VERSION.startsWith('v')) {
72-
throw new TypeError(
73-
'provided version not in correct format, expected vX.X.X'
74-
);
75-
}
76-
77-
console.log(`Listing version ${VERSION}`);
78-
79-
const versionDirectory = await listR2DirectoryRecursive(
80-
s3Client,
81-
PROD_BUCKET,
82-
`${RELEASE_DIR}${VERSION}/`
83-
);
84-
85-
for (const [k, v] of versionDirectory.entries()) {
86-
directories.set(k, v);
87-
}
88-
}
89-
9067
const latestVersions = await getLatestVersionMapping(
9168
s3Client,
9269
directories.get(RELEASE_DIR),
@@ -95,6 +72,8 @@ const latestVersions = await getLatestVersionMapping(
9572

9673
await addSymlinksToDirectoryCache(s3Client, directories, latestVersions);
9774

98-
// Update keys in the directory cache
75+
// Update keys in the directory cache. These intentionally do not have an
76+
// expiration set since they were made with info that the worker doesn't have
77+
// when deployed.
9978
console.log(`Writing ${directories.size} keys to directory cache...`);
10079
await writeKeysToKv(cfClient, DIRECTORY_CACHE_NAMESPACE_ID, directories);

‎scripts/utils/addSymlinksToDirectoryCache.mjs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,10 +124,10 @@ export async function addStaticFileSymlinksToCache(
124124
*/
125125
const fileSymlinks = JSON.parse(fileSymlinksFile);
126126

127-
// This file is an exception for two reasons:
128-
// 1. It's dynamic and points to the latest Node version
129-
// 2. It's handled specially and is technically in the root directory,
130-
// which we should never be listing thus don't need to add it to the
127+
// These files are an exception for two reasons:
128+
// 1. They're dynamic and point to the latest Node version
129+
// 2. They're handled specially and are technically in the root directory,
130+
// which we should never be listing thus don't need to add them to the
131131
// directory cache.
132132
// So, let's remove them then add them back once we're done handling the rest
133133
// of the static file symlinks.

‎src/constants/cache.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,8 @@ export const CACHE_HEADERS = {
44
mutable: 'public, max-age=3600, s-maxage=14400',
55
failure: 'private, no-cache, no-store, max-age=0, must-revalidate',
66
};
7+
8+
/**
9+
* Ttl for elements in the KV cache in seconds. This is completely arbitrary
10+
*/
11+
export const KV_CACHE_TTL = 60 * 60 * 6;

0 commit comments

Comments
 (0)