|
| 1 | +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. |
| 2 | + |
| 3 | +/** |
| 4 | + * A chunked upload to S3 finds its object key in the upload session row, on |
| 5 | + * any process. |
| 6 | + * |
| 7 | + * S3 addresses a multipart upload by `(Key, UploadId)`, but the |
| 8 | + * `IStorageService` chunk and completion methods carry the `uploadId` alone. |
| 9 | + * The S3 adapter answers the key from its own `uploadId → key` map, which |
| 10 | + * `setUploadKey` fills. Two ways that map came up empty: |
| 11 | + * |
| 12 | + * 1. The plugin registers the SWAPPABLE proxy as the kernel's `storage` |
| 13 | + * service, and the proxy did not forward `setUploadKey`, so the init |
| 14 | + * door's call never reached the adapter. Chunk 0 answered 500 on a single |
| 15 | + * replica. |
| 16 | + * 2. The map lives in one process. A chunk or completion served by another |
| 17 | + * replica (or by the same replica after a restart or an adapter swap) |
| 18 | + * had no entry, whatever the init door did. |
| 19 | + * |
| 20 | + * The durable record of the key is `sys_upload_session.key`, written by the |
| 21 | + * init door. The chunk and completion doors read that row before they reach |
| 22 | + * the backend, so they hand its key to the adapter on every call. These cases |
| 23 | + * drive the doors against an S3-compatible stand-in that, like S3, refuses a |
| 24 | + * part or a completion addressed to the wrong key. |
| 25 | + */ |
| 26 | + |
| 27 | +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; |
| 28 | + |
| 29 | +const fakeS3 = vi.hoisted(() => { |
| 30 | + const objects = new Map<string, Buffer>(); |
| 31 | + const multipart = new Map<string, { key: string; parts: Map<number, Buffer> }>(); |
| 32 | + let uploadSeq = 0; |
| 33 | + |
| 34 | + function noSuchUpload(): Error { |
| 35 | + const err = new Error('NoSuchUpload') as Error & { name: string; $metadata: { httpStatusCode: number } }; |
| 36 | + err.name = 'NoSuchUpload'; |
| 37 | + err.$metadata = { httpStatusCode: 404 }; |
| 38 | + return err; |
| 39 | + } |
| 40 | + |
| 41 | + /** S3 matches BOTH halves of the address: an UploadId under another Key is no upload at all. */ |
| 42 | + function upload(input: { Key?: string; UploadId?: string }) { |
| 43 | + const found = input.UploadId ? multipart.get(input.UploadId) : undefined; |
| 44 | + if (!found || found.key !== input.Key) throw noSuchUpload(); |
| 45 | + return found; |
| 46 | + } |
| 47 | + |
| 48 | + function reset(): void { |
| 49 | + objects.clear(); |
| 50 | + multipart.clear(); |
| 51 | + uploadSeq = 0; |
| 52 | + } |
| 53 | + |
| 54 | + return { objects, multipart, upload, reset, nextUploadId: () => `s3-upload-${++uploadSeq}` }; |
| 55 | +}); |
| 56 | + |
| 57 | +vi.mock('@aws-sdk/client-s3', () => { |
| 58 | + class Command { |
| 59 | + constructor(public readonly input: Record<string, any>) {} |
| 60 | + } |
| 61 | + class CreateMultipartUploadCommand extends Command {} |
| 62 | + class UploadPartCommand extends Command {} |
| 63 | + class CompleteMultipartUploadCommand extends Command {} |
| 64 | + class AbortMultipartUploadCommand extends Command {} |
| 65 | + |
| 66 | + class S3Client { |
| 67 | + constructor(_config: unknown) {} |
| 68 | + async send(command: any): Promise<any> { |
| 69 | + const input = command.input; |
| 70 | + if (command instanceof CreateMultipartUploadCommand) { |
| 71 | + const uploadId = fakeS3.nextUploadId(); |
| 72 | + fakeS3.multipart.set(uploadId, { key: input.Key, parts: new Map() }); |
| 73 | + return { UploadId: uploadId }; |
| 74 | + } |
| 75 | + if (command instanceof UploadPartCommand) { |
| 76 | + fakeS3.upload(input).parts.set(input.PartNumber, input.Body); |
| 77 | + return { ETag: `"etag-${input.PartNumber}"` }; |
| 78 | + } |
| 79 | + if (command instanceof CompleteMultipartUploadCommand) { |
| 80 | + const found = fakeS3.upload(input); |
| 81 | + const listed: Array<{ PartNumber: number }> = input.MultipartUpload.Parts; |
| 82 | + fakeS3.objects.set(input.Key, Buffer.concat(listed.map((p) => found.parts.get(p.PartNumber)!))); |
| 83 | + fakeS3.multipart.delete(input.UploadId); |
| 84 | + return {}; |
| 85 | + } |
| 86 | + if (command instanceof AbortMultipartUploadCommand) { |
| 87 | + fakeS3.upload(input); |
| 88 | + fakeS3.multipart.delete(input.UploadId); |
| 89 | + return {}; |
| 90 | + } |
| 91 | + throw new Error(`S3 stand-in: unhandled command ${command.constructor.name}`); |
| 92 | + } |
| 93 | + } |
| 94 | + |
| 95 | + return { |
| 96 | + S3Client, |
| 97 | + CreateMultipartUploadCommand, |
| 98 | + UploadPartCommand, |
| 99 | + CompleteMultipartUploadCommand, |
| 100 | + AbortMultipartUploadCommand, |
| 101 | + }; |
| 102 | +}); |
| 103 | + |
| 104 | +import type { IHttpRequest, IHttpResponse, IStorageService, RouteHandler } from '@objectstack/spec/contracts'; |
| 105 | +import { promises as fs } from 'node:fs'; |
| 106 | +import { join } from 'node:path'; |
| 107 | +import { tmpdir } from 'node:os'; |
| 108 | +import { S3StorageAdapter } from './s3-storage-adapter.js'; |
| 109 | +import { LocalStorageAdapter } from './local-storage-adapter.js'; |
| 110 | +import { SwappableStorageService } from './swappable-storage-service.js'; |
| 111 | +import { StorageMetadataStore } from './metadata-store.js'; |
| 112 | +import { registerStorageRoutes } from './storage-routes.js'; |
| 113 | + |
| 114 | +const BASE = '/api/v1/storage'; |
| 115 | +const INIT = `${BASE}/upload/chunked`; |
| 116 | +const CHUNK = `${BASE}/upload/chunked/:uploadId/chunk/:chunkIndex`; |
| 117 | +const COMPLETE = `${BASE}/upload/chunked/:uploadId/complete`; |
| 118 | + |
| 119 | +/** The smallest chunk the init door accepts (5 MiB), so two chunks are one full and one short. */ |
| 120 | +const CHUNK_SIZE = 5 * 1024 * 1024; |
| 121 | +const FIRST = Buffer.alloc(CHUNK_SIZE, 0x61); |
| 122 | +const SECOND = Buffer.from('tail'); |
| 123 | + |
| 124 | +/** |
| 125 | + * The stored object is the whole file, byte for byte. Compared with |
| 126 | + * `Buffer.equals` rather than `toEqual`, which walks 5 MiB one element at a time. |
| 127 | + */ |
| 128 | +function expectWhole(stored: Buffer | undefined): void { |
| 129 | + expect(stored?.length).toBe(FIRST.length + SECOND.length); |
| 130 | + expect(stored?.equals(Buffer.concat([FIRST, SECOND]))).toBe(true); |
| 131 | +} |
| 132 | + |
| 133 | +type Res = IHttpResponse & { _status: number; _json: any }; |
| 134 | + |
| 135 | +function createMockRes(): Res { |
| 136 | + const res: any = { |
| 137 | + _status: 200, |
| 138 | + _json: null, |
| 139 | + json(data: any) { res._json = data; }, |
| 140 | + send(data: any) { res._sent = data; }, |
| 141 | + status(code: number) { res._status = code; return res; }, |
| 142 | + header() { return res; }, |
| 143 | + }; |
| 144 | + return res; |
| 145 | +} |
| 146 | + |
| 147 | +function createMockReq(overrides: Record<string, unknown>): IHttpRequest { |
| 148 | + return { params: {}, query: {}, body: undefined, headers: {}, method: 'GET', path: '/', ...overrides } as IHttpRequest; |
| 149 | +} |
| 150 | + |
| 151 | +/** |
| 152 | + * One server process: its own adapter instance (so its own in-process state) |
| 153 | + * mounted behind its own swappable proxy, the way `StorageServicePlugin` |
| 154 | + * composes it. Replicas share the metadata store (the database) and the bucket. |
| 155 | + */ |
| 156 | +function replica(store: StorageMetadataStore, inner: IStorageService) { |
| 157 | + const routes = new Map<string, RouteHandler>(); |
| 158 | + const server = { |
| 159 | + get: (path: string, h: RouteHandler) => { routes.set(`GET:${path}`, h); }, |
| 160 | + post: (path: string, h: RouteHandler) => { routes.set(`POST:${path}`, h); }, |
| 161 | + put: (path: string, h: RouteHandler) => { routes.set(`PUT:${path}`, h); }, |
| 162 | + delete: () => {}, |
| 163 | + patch: () => {}, |
| 164 | + use: () => {}, |
| 165 | + }; |
| 166 | + registerStorageRoutes(server as any, new SwappableStorageService(inner), store, { basePath: BASE }); |
| 167 | + const call = async (method: string, path: string, req: Record<string, unknown>): Promise<Res> => { |
| 168 | + const res = createMockRes(); |
| 169 | + await routes.get(`${method}:${path}`)!(createMockReq(req), res); |
| 170 | + return res; |
| 171 | + }; |
| 172 | + return { |
| 173 | + init: () => |
| 174 | + call('POST', INIT, { |
| 175 | + body: { filename: 'big.bin', mimeType: 'application/octet-stream', totalSize: FIRST.length + SECOND.length, chunkSize: CHUNK_SIZE }, |
| 176 | + }), |
| 177 | + chunk: (uploadId: string, resumeToken: string, chunkIndex: number, bytes: Buffer) => |
| 178 | + call('PUT', CHUNK, { |
| 179 | + params: { uploadId, chunkIndex: String(chunkIndex) }, |
| 180 | + headers: { 'x-resume-token': resumeToken }, |
| 181 | + rawBody: async () => bytes, |
| 182 | + }), |
| 183 | + complete: (uploadId: string, parts: Array<{ chunkIndex: number; eTag: string }>) => |
| 184 | + call('POST', COMPLETE, { params: { uploadId }, body: { uploadId, parts } }), |
| 185 | + }; |
| 186 | +} |
| 187 | + |
| 188 | +const s3 = () => new S3StorageAdapter({ bucket: 'uploads', region: 'us-east-1', keyPrefix: null }); |
| 189 | + |
| 190 | +/** Drive one upload: init on `a`, chunk 0 on `b`, chunk 1 and completion on `c`. */ |
| 191 | +async function drive(a: ReturnType<typeof replica>, b: ReturnType<typeof replica>, c: ReturnType<typeof replica>) { |
| 192 | + const init = await a.init(); |
| 193 | + expect(init._status, JSON.stringify(init._json)).toBe(200); |
| 194 | + const { uploadId, resumeToken } = init._json.data; |
| 195 | + |
| 196 | + const first = await b.chunk(uploadId, resumeToken, 0, FIRST); |
| 197 | + expect(first._status, JSON.stringify(first._json)).toBe(200); |
| 198 | + const second = await c.chunk(uploadId, resumeToken, 1, SECOND); |
| 199 | + expect(second._status, JSON.stringify(second._json)).toBe(200); |
| 200 | + |
| 201 | + const done = await c.complete(uploadId, [ |
| 202 | + { chunkIndex: 0, eTag: first._json.data.eTag }, |
| 203 | + { chunkIndex: 1, eTag: second._json.data.eTag }, |
| 204 | + ]); |
| 205 | + expect(done._status, JSON.stringify(done._json)).toBe(200); |
| 206 | + return done._json.data as { fileId: string; key: string; size: number }; |
| 207 | +} |
| 208 | + |
| 209 | +beforeEach(() => { |
| 210 | + fakeS3.reset(); |
| 211 | +}); |
| 212 | + |
| 213 | +describe('a chunked upload through the swappable proxy reaches the S3 object it started', () => { |
| 214 | + it('completes on a single replica', async () => { |
| 215 | + const one = replica(new StorageMetadataStore(null), s3()); |
| 216 | + |
| 217 | + const done = await drive(one, one, one); |
| 218 | + |
| 219 | + expect(done.size).toBe(FIRST.length + SECOND.length); |
| 220 | + expectWhole(fakeS3.objects.get(done.key)); |
| 221 | + expect(fakeS3.multipart.size).toBe(0); |
| 222 | + }); |
| 223 | + |
| 224 | + it('resumes on a fresh adapter instance: another process finds the key in the session row', async () => { |
| 225 | + // One database and one bucket; three processes, none sharing memory. The |
| 226 | + // replica that serves a chunk or the completion has never seen the init. |
| 227 | + const store = new StorageMetadataStore(null); |
| 228 | + const a = replica(store, s3()); |
| 229 | + const b = replica(store, s3()); |
| 230 | + const c = replica(store, s3()); |
| 231 | + |
| 232 | + const done = await drive(a, b, c); |
| 233 | + |
| 234 | + expectWhole(fakeS3.objects.get(done.key)); |
| 235 | + expect(fakeS3.multipart.size).toBe(0); |
| 236 | + // The key the object landed under is the one the session row recorded. |
| 237 | + const file = await store.getFile(done.fileId); |
| 238 | + expect(file?.key).toBe(done.key); |
| 239 | + expect(file?.status).toBe('committed'); |
| 240 | + }); |
| 241 | +}); |
| 242 | + |
| 243 | +describe('adapters that do not track upload keys are untouched', () => { |
| 244 | + let rootDir: string; |
| 245 | + |
| 246 | + afterEach(async () => { |
| 247 | + if (rootDir) await fs.rm(rootDir, { recursive: true, force: true }); |
| 248 | + }); |
| 249 | + |
| 250 | + it('local: the assembled file holds the bytes sent, across two adapter instances on one root', async () => { |
| 251 | + rootDir = join(tmpdir(), `os-chunk-session-key-${Date.now()}-${Math.random().toString(36).slice(2)}`); |
| 252 | + await fs.mkdir(rootDir, { recursive: true }); |
| 253 | + const store = new StorageMetadataStore(null); |
| 254 | + const local = () => new LocalStorageAdapter({ rootDir, signingSecret: 'test-secret' }); |
| 255 | + const reader = local(); |
| 256 | + |
| 257 | + const done = await drive(replica(store, local()), replica(store, local()), replica(store, local())); |
| 258 | + |
| 259 | + expectWhole(await reader.download(done.key)); |
| 260 | + }); |
| 261 | + |
| 262 | + it('an adapter with no setUploadKey receives exactly the contract calls, with the arguments it did before', async () => { |
| 263 | + const calls: unknown[][] = []; |
| 264 | + const parts = new Map<string, Buffer[]>(); |
| 265 | + const memory: IStorageService = { |
| 266 | + upload: async () => {}, |
| 267 | + download: async () => Buffer.alloc(0), |
| 268 | + delete: async () => {}, |
| 269 | + exists: async () => false, |
| 270 | + getInfo: async (key) => ({ key, size: 0, lastModified: new Date(0) }), |
| 271 | + initiateChunkedUpload: async (key) => { |
| 272 | + calls.push(['initiateChunkedUpload', key]); |
| 273 | + parts.set('mem-1', []); |
| 274 | + return 'mem-1'; |
| 275 | + }, |
| 276 | + uploadChunk: async (uploadId, partNumber, data) => { |
| 277 | + calls.push(['uploadChunk', uploadId, partNumber, data.length]); |
| 278 | + parts.get(uploadId)!.push(data); |
| 279 | + return `m${partNumber}`; |
| 280 | + }, |
| 281 | + completeChunkedUpload: async (uploadId, listed) => { |
| 282 | + calls.push(['completeChunkedUpload', uploadId, listed]); |
| 283 | + return 'assembled-key'; |
| 284 | + }, |
| 285 | + }; |
| 286 | + |
| 287 | + const one = replica(new StorageMetadataStore(null), memory); |
| 288 | + const done = await drive(one, one, one); |
| 289 | + |
| 290 | + expect(done.key).toBe('assembled-key'); |
| 291 | + expect(calls).toEqual([ |
| 292 | + ['initiateChunkedUpload', expect.stringMatching(/^user\/.+\.bin$/)], |
| 293 | + ['uploadChunk', 'mem-1', 1, FIRST.length], |
| 294 | + ['uploadChunk', 'mem-1', 2, SECOND.length], |
| 295 | + ['completeChunkedUpload', 'mem-1', [{ partNumber: 1, eTag: 'm1' }, { partNumber: 2, eTag: 'm2' }]], |
| 296 | + ]); |
| 297 | + }); |
| 298 | +}); |
0 commit comments