diff --git a/handwritten/storage/src/file.ts b/handwritten/storage/src/file.ts index 9cbd6ee7578..45fb6df758a 100644 --- a/handwritten/storage/src/file.ts +++ b/handwritten/storage/src/file.ts @@ -2027,31 +2027,41 @@ class File extends ServiceObject { retryOptions.autoRetry = false; } - resumableUpload.createURI( - { - authClient: this.storage.storageTransport.authClient, - apiEndpoint: this.storage.apiEndpoint, - bucket: this.bucket.name, - file: this.name, - generation: this.generation, - key: this.encryptionKey === null ? undefined : this.encryptionKey, - kmsKeyName: this.kmsKeyName, - metadata: options.metadata, - offset: options.offset, - origin: options.origin, - predefinedAcl: options.predefinedAcl, - private: options.private, - public: options.public, - userProject: options.userProject || this.userProject, - retryOptions: retryOptions, - params: options?.preconditionOpts || this.instancePreconditionOpts, - universeDomain: this.bucket.storage.universeDomain, - useAuthWithCustomEndpoint: this.storage.useAuthWithCustomEndpoint, - [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], - }, - callback!, - ); - this.storage.retryOptions.autoRetry = this.instanceRetryValue; + void (async () => { + try { + const apiEndpoint = + (typeof this.storage.getEndpoint === 'function' + ? await this.storage.getEndpoint() + : undefined) || this.storage['apiEndpoint']; + resumableUpload.createURI( + { + authClient: this.storage.storageTransport.authClient, + apiEndpoint, + bucket: this.bucket.name, + file: this.name, + generation: this.generation, + key: this.encryptionKey === null ? undefined : this.encryptionKey, + kmsKeyName: this.kmsKeyName, + metadata: options.metadata, + offset: options.offset, + origin: options.origin, + predefinedAcl: options.predefinedAcl, + private: options.private, + public: options.public, + userProject: options.userProject || this.userProject, + retryOptions: {...retryOptions}, + params: options?.preconditionOpts || this.instancePreconditionOpts, + universeDomain: this.bucket.storage.universeDomain, + [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], + }, + callback!, + ); + } catch (err) { + callback!(err as Error); + } finally { + this.storage.retryOptions.autoRetry = this.instanceRetryValue; + } + })(); } /** @@ -2437,7 +2447,7 @@ class File extends ServiceObject { ); if (options.resumable === false) { - this.startSimpleUpload_( + void this.startSimpleUpload_( fileWriteStream, options as CreateWriteStreamOptionsInternal, ); @@ -3272,14 +3282,26 @@ class File extends ServiceObject { options.signingEndpoint, ); const signatureHex = Buffer.from(signature, 'base64').toString('hex'); - const universe = this.parent.storage.universeDomain; fields['policy'] = policyBase64; fields['x-goog-signature'] = signatureHex; let url: string; - if (this.storage.customEndpoint) { - url = this.storage.apiEndpoint; + const EMULATOR_HOST = process.env.STORAGE_EMULATOR_HOST; + const endpoint = + (typeof this.storage.getEndpoint === 'function' + ? await this.storage.getEndpoint() + : undefined) || this.storage['apiEndpoint']; + const universe = + this.parent.storage.universeDomain || + this.storage.universeDomain || + 'googleapis.com'; + const defaultEndpoint = `https://storage.${universe}`; + + if (endpoint !== defaultEndpoint && typeof EMULATOR_HOST === 'string') { + url = `${endpoint}/${this.bucket.name}`; + } else if (endpoint !== defaultEndpoint) { + url = endpoint; } else if (options.virtualHostedStyle) { url = `https://${this.bucket.name}.storage.${universe}/`; } else if (options.bucketBoundHostname) { @@ -3610,11 +3632,6 @@ class File extends ServiceObject { undefined, callback, ); - const baseUrl = this.storage.apiEndpoint.startsWith('http') - ? this.storage.apiEndpoint - : `https://${this.storage.apiEndpoint}`; - - const url = `${baseUrl}/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`; const gaxios = new Gaxios(); const storageInterceptors = this.storage?.interceptors || []; @@ -3627,6 +3644,16 @@ class File extends ServiceObject { // eslint-disable-next-line @typescript-eslint/no-floating-promises (async () => { try { + const endpoint = + (typeof this.storage.getEndpoint === 'function' + ? await this.storage.getEndpoint() + : undefined) || this.storage['apiEndpoint']; + const baseUrl = endpoint.startsWith('http') + ? endpoint + : `https://${endpoint}`; + + const url = `${baseUrl}/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`; + await gaxios.request({ method: 'GET', url, @@ -3823,7 +3850,7 @@ class File extends ServiceObject { * ``` */ publicUrl(): string { - return `${this.storage.apiEndpoint}/${ + return `${this.storage['apiEndpoint']}/${ this.bucket.name }/${encodeURIComponent(this.name)}`; } @@ -4755,59 +4782,64 @@ class File extends ServiceObject { ) { retryOptions.autoRetry = false; } - const cfg = { - authClient: this.storage.storageTransport.authClient, - apiEndpoint: this.storage.apiEndpoint, - bucket: this.bucket.name, - file: this.name, - generation: this.generation, - isPartialUpload: options.isPartialUpload, - key: this.encryptionKey === null ? undefined : this.encryptionKey, - kmsKeyName: this.kmsKeyName, - metadata: options.metadata, - offset: options.offset, - predefinedAcl: options.predefinedAcl, - private: options.private, - public: options.public, - uri: options.uri, - userProject: options.userProject || this.userProject, - retryOptions: {...retryOptions}, - params: options?.preconditionOpts || this.instancePreconditionOpts, - chunkSize: options?.chunkSize, - highWaterMark: options?.highWaterMark, - universeDomain: this.bucket.storage.universeDomain, - useAuthWithCustomEndpoint: this.storage.useAuthWithCustomEndpoint, - [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], - }; + const uploadRetryOptions = {...retryOptions}; + this.storage.retryOptions.autoRetry = this.instanceRetryValue; - let uploadStream: resumableUpload.Upload; + void (async () => { + let uploadStream: resumableUpload.Upload; - try { - uploadStream = resumableUpload.upload(cfg); - } catch (error) { - dup.destroy(error as Error); - this.storage.retryOptions.autoRetry = this.instanceRetryValue; - return; - } + try { + const apiEndpoint = + (typeof this.storage.getEndpoint === 'function' + ? await this.storage.getEndpoint() + : undefined) || this.storage['apiEndpoint']; + const cfg = { + authClient: this.storage.storageTransport.authClient, + apiEndpoint, + bucket: this.bucket.name, + file: this.name, + generation: this.generation, + isPartialUpload: options.isPartialUpload, + key: this.encryptionKey === null ? undefined : this.encryptionKey, + kmsKeyName: this.kmsKeyName, + metadata: options.metadata, + offset: options.offset, + predefinedAcl: options.predefinedAcl, + private: options.private, + public: options.public, + uri: options.uri, + userProject: options.userProject || this.userProject, + retryOptions: uploadRetryOptions, + params: options?.preconditionOpts || this.instancePreconditionOpts, + chunkSize: options?.chunkSize, + highWaterMark: options?.highWaterMark, + universeDomain: this.bucket.storage.universeDomain, + [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY], + }; + uploadStream = resumableUpload.upload(cfg); + } catch (error) { + dup.destroy(error as Error); + return; + } - uploadStream - .on('response', resp => { - dup.emit('response', resp); - }) - .on('uri', uri => { - dup.emit('uri', uri); - }) - .on('metadata', metadata => { - this.metadata = metadata; - dup.emit('metadata'); - }) - .on('finish', () => { - dup.emit('complete'); - }) - .on('progress', evt => dup.emit('progress', evt)); + uploadStream + .on('response', resp => { + dup.emit('response', resp); + }) + .on('uri', uri => { + dup.emit('uri', uri); + }) + .on('metadata', metadata => { + this.metadata = metadata; + dup.emit('metadata'); + }) + .on('finish', () => { + dup.emit('complete'); + }) + .on('progress', evt => dup.emit('progress', evt)); - dup.setWritable(uploadStream); - this.storage.retryOptions.autoRetry = this.instanceRetryValue; + dup.setWritable(uploadStream); + })(); } /** @@ -4820,13 +4852,26 @@ class File extends ServiceObject { * * @private */ - startSimpleUpload_( + async startSimpleUpload_( dup: Duplexify, options: CreateWriteStreamOptionsInternal = {}, - ): void { + ): Promise { options.metadata ??= {}; - const apiEndpoint = this.storage.apiEndpoint; + const writeStream = new ProgressStream(); + writeStream.on('progress', evt => dup.emit('progress', evt)); + dup.setWritable(writeStream); + + let apiEndpoint: string; + try { + apiEndpoint = + (typeof this.storage.getEndpoint === 'function' + ? await this.storage.getEndpoint() + : undefined) || this.storage['apiEndpoint']; + } catch (err) { + dup.destroy(err as Error); + return; + } const bucketName = this.bucket.name; const url = `${apiEndpoint}/upload/storage/v1/b/${bucketName}/o`; @@ -4873,10 +4918,6 @@ class File extends ServiceObject { options.preconditionOpts, ); - const writeStream = new ProgressStream(); - writeStream.on('progress', evt => dup.emit('progress', evt)); - dup.setWritable(writeStream); - reqOpts.multipart = [ { headers: new Headers({'Content-Type': 'application/json'}), @@ -4914,7 +4955,7 @@ class File extends ServiceObject { dup.emit('response', resp); dup.emit('complete'); }) - .catch(err => dup.destroy(err)); + ?.catch(err => dup.destroy(err)); } disableAutoRetryConditionallyIdempotent_( diff --git a/handwritten/storage/src/nodejs-common/util.ts b/handwritten/storage/src/nodejs-common/util.ts index 977c955d1fc..7f3b26c75f3 100644 --- a/handwritten/storage/src/nodejs-common/util.ts +++ b/handwritten/storage/src/nodejs-common/util.ts @@ -104,11 +104,13 @@ export interface MakeAuthenticatedRequestFactoryConfig extends Omit< /** * If true, just return the provided request options. Default: false. + * @deprecated */ customEndpoint?: boolean; /** * If true, will authenticate when using a custom endpoint. Default: false. + * @deprecated */ useAuthWithCustomEndpoint?: boolean; diff --git a/handwritten/storage/src/resumable-upload.ts b/handwritten/storage/src/resumable-upload.ts index de3661951bf..f55d6b73c48 100644 --- a/handwritten/storage/src/resumable-upload.ts +++ b/handwritten/storage/src/resumable-upload.ts @@ -18,7 +18,6 @@ import { GaxiosResponse, GaxiosError, } from 'gaxios'; -import * as gaxios from 'gaxios'; import { DEFAULT_UNIVERSE, GoogleAuth, @@ -247,6 +246,7 @@ export interface UploadConfig extends Pick { /** * Controls whether or not to use authentication when using a custom endpoint. + * @deprecated */ useAuthWithCustomEndpoint?: boolean; @@ -385,33 +385,6 @@ export class Upload extends Writable { this.apiEndpoint = `https://storage.${universe}`; if (cfg.apiEndpoint && cfg.apiEndpoint !== this.apiEndpoint) { this.apiEndpoint = this.sanitizeEndpoint(cfg.apiEndpoint); - - const hostname = new URL(this.apiEndpoint).hostname; - - // check if it is a domain of a known universe - const isDomain = hostname === universe; - const isDefaultUniverseDomain = hostname === DEFAULT_UNIVERSE; - - // check if it is a subdomain of a known universe - // by checking a last (universe's length + 1) of a hostname - const isSubDomainOfUniverse = - hostname.slice(-(universe.length + 1)) === `.${universe}`; - const isSubDomainOfDefaultUniverse = - hostname.slice(-(DEFAULT_UNIVERSE.length + 1)) === - `.${DEFAULT_UNIVERSE}`; - - if ( - !isDomain && - !isDefaultUniverseDomain && - !isSubDomainOfUniverse && - !isSubDomainOfDefaultUniverse - ) { - // a custom, non-universe domain, - // use gaxios only if useAuthWithCustomEndpoint is not true - if (cfg.useAuthWithCustomEndpoint !== true) { - this.authClient = gaxios; - } - } } this.baseURI = `${this.apiEndpoint}/upload/storage/v1/b`; diff --git a/handwritten/storage/src/signer.ts b/handwritten/storage/src/signer.ts index ac7d1c1b659..ae59b723ebe 100644 --- a/handwritten/storage/src/signer.ts +++ b/handwritten/storage/src/signer.ts @@ -204,12 +204,17 @@ export class URLSigner { ); } - return promise.then(query => { + return promise.then(async query => { query = Object.assign(query, cfg.queryParams); - const signedUrl = new url.URL( - cfg.host?.toString() || config.cname || this.storage.apiEndpoint, - ); + const endpoint = + config.host?.toString() || + config.cname || + (typeof this.storage.getEndpoint === 'function' + ? await this.storage.getEndpoint() + : undefined) || + this.storage['apiEndpoint']; + const signedUrl = new url.URL(endpoint); signedUrl.pathname = this.getResourcePath( !!config.cname, @@ -284,17 +289,6 @@ export class URLSigner { } const extensionHeaders = Object.assign({}, config.extensionHeaders); - const fqdn = new url.URL( - config.host?.toString() || config.cname || this.storage.apiEndpoint, - ); - extensionHeaders.host = fqdn.hostname; - if (config.contentMd5) { - extensionHeaders['content-md5'] = config.contentMd5; - } - if (config.contentType) { - extensionHeaders['content-type'] = config.contentType; - } - let contentSha256: string; const sha256Header = extensionHeaders['x-goog-content-sha256']; if (sha256Header) { @@ -307,17 +301,32 @@ export class URLSigner { contentSha256 = sha256Header; } - const signedHeaders = Object.keys(extensionHeaders) - .map(header => header.toLowerCase()) - .sort() - .join(';'); - - const extensionHeadersString = this.getCanonicalHeaders(extensionHeaders); - const datestamp = formatAsUTCISO(config.accessibleAt); const credentialScope = `${datestamp}/auto/storage/goog4_request`; const sign = async () => { + if (!config.host && !config.cname) { + config.host = + (typeof this.storage.getEndpoint === 'function' + ? await this.storage.getEndpoint() + : undefined) || this.storage['apiEndpoint']; + } + const fqdn = new url.URL(config.host?.toString() || config.cname!); + extensionHeaders.host = fqdn.hostname; + if (config.contentMd5) { + extensionHeaders['content-md5'] = config.contentMd5; + } + if (config.contentType) { + extensionHeaders['content-type'] = config.contentType; + } + + const signedHeaders = Object.keys(extensionHeaders) + .map(header => header.toLowerCase()) + .sort() + .join(';'); + + const extensionHeadersString = this.getCanonicalHeaders(extensionHeaders); + const credentials = await this.auth.getCredentials(); const credential = `${credentials.client_email}/${credentialScope}`; const dateISO = formatAsUTCISO( diff --git a/handwritten/storage/src/storage.ts b/handwritten/storage/src/storage.ts index 85f60f8755f..35bc9a8f14a 100644 --- a/handwritten/storage/src/storage.ts +++ b/handwritten/storage/src/storage.ts @@ -33,6 +33,7 @@ import { DEFAULT_UNIVERSE, GoogleAuth, GoogleAuthOptions, + PassThroughClient, } from 'google-auth-library'; import { StorageQueryParameters, @@ -121,6 +122,9 @@ export interface StorageOptions extends Omit { token?: string; timeout?: number; // http.request.options.timeout userAgent?: string; + /** + * @deprecated + */ useAuthWithCustomEndpoint?: boolean; } @@ -650,11 +654,16 @@ export class Storage { crc32cGenerator: CRC32CValidatorGenerator; projectId?: string; - apiEndpoint: string; storageTransport: StorageTransport; interceptors: GaxiosInterceptor[]; universeDomain: string; + /** + * @deprecated + */ customEndpoint = false; + /** + * @deprecated + */ useAuthWithCustomEndpoint?: boolean; name = ''; baseUrl = ''; @@ -670,6 +679,8 @@ export class Storage { } retryOptions: RetryOptions; + protected apiEndpoint: string; + #hasCustomEndpoint = false; /** * @callback Crc32cGeneratorToStringCallback @@ -782,7 +793,6 @@ export class Storage { * @property {number} [timeout] The amount of time in milliseconds to wait per http request before timing out. * @property {object[]} [interceptors_] Array of custom request interceptors to be returned in the order they were assigned. * @property {string} [apiEndpoint = storage.google.com] The API endpoint of the service used to make requests. - * @property {boolean} [useAuthWithCustomEndpoint = false] Controls whether or not to use authentication when using a custom endpoint. * @property {Crc32cGeneratorCallback} [callback] A function that generates a CRC32C Validator. Defaults to {@link CRC32C} */ /** @@ -855,18 +865,29 @@ export class Storage { const universe = options.universeDomain || DEFAULT_UNIVERSE; let apiEndpoint = `https://storage.${universe}`; + let hasCustomEndpoint = false; this.projectId = options.projectId; // Note: EMULATOR_HOST is an experimental configuration variable. Use apiEndpoint instead. const EMULATOR_HOST = process.env.STORAGE_EMULATOR_HOST; if (typeof EMULATOR_HOST === 'string') { apiEndpoint = Storage.sanitizeEndpoint(EMULATOR_HOST); - this.customEndpoint = true; + hasCustomEndpoint = true; + if ( + !options.authClient && + !options.keyFilename && + !options.keyFile && + !options.credentials + ) { + options = Object.assign({}, options, { + authClient: new PassThroughClient(), + }); + } } if (options.apiEndpoint && options.apiEndpoint !== apiEndpoint) { apiEndpoint = Storage.sanitizeEndpoint(options.apiEndpoint); - this.customEndpoint = true; + hasCustomEndpoint = true; } options = Object.assign({}, options, {apiEndpoint}); @@ -902,8 +923,6 @@ export class Storage { : IDEMPOTENCY_STRATEGY_DEFAULT, }, baseUrl: this.baseUrl, - customEndpoint: this.customEndpoint, - useAuthWithCustomEndpoint: options?.useAuthWithCustomEndpoint, scopes: [ 'https://www.googleapis.com/auth/iam', 'https://www.googleapis.com/auth/cloud-platform', @@ -912,8 +931,8 @@ export class Storage { packageJson: getPackageJSON(), }; - this.apiEndpoint = options.apiEndpoint!; - this.useAuthWithCustomEndpoint = options.useAuthWithCustomEndpoint; + this.apiEndpoint = apiEndpoint; + this.#hasCustomEndpoint = hasCustomEndpoint; /** * Reference to {@link Storage.acl}. @@ -939,6 +958,19 @@ export class Storage { this.getHmacKeysStream = paginator.streamify('getHmacKeys'); } + async getEndpoint(): Promise { + if (!this.#hasCustomEndpoint) { + const universe = + typeof this.storageTransport?.authClient?.getUniverseDomain === + 'function' + ? await this.storageTransport.authClient.getUniverseDomain() + : this.universeDomain; + this.universeDomain = universe; + this.apiEndpoint = Storage.sanitizeEndpoint(`storage.${universe}`); + } + return this.apiEndpoint; + } + private static sanitizeEndpoint(url: string) { if (!PROTOCOL_REGEX.test(url)) { url = `https://${url}`; @@ -1796,5 +1828,5 @@ paginator.extend(Storage, ['getBuckets', 'getHmacKeys']); * that a callback is omitted. */ promisifyAll(Storage, { - exclude: ['bucket', 'channel', 'hmacKey'], + exclude: ['bucket', 'channel', 'hmacKey', 'getEndpoint'], }); diff --git a/handwritten/storage/src/transfer-manager.ts b/handwritten/storage/src/transfer-manager.ts index ac0c080bac4..e3f366f0ebd 100644 --- a/handwritten/storage/src/transfer-manager.ts +++ b/handwritten/storage/src/transfer-manager.ts @@ -214,7 +214,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { this.bucket = bucket; this.fileName = fileName; this.baseUrl = `https://${bucket.name}.${ - new URL(this.bucket.storage.apiEndpoint).hostname + new URL(this.bucket.storage['apiEndpoint']).hostname }/${fileName}`; this.xmlBuilder = new XMLBuilder({arrayNodeName: 'Part'}); this.xmlParser = new XMLParser(); @@ -254,6 +254,17 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { return headers; } + async #getBaseUrl(): Promise { + const endpoint = + (typeof this.bucket.storage.getEndpoint === 'function' + ? await this.bucket.storage.getEndpoint() + : undefined) || this.bucket.storage['apiEndpoint']; + this.baseUrl = `https://${this.bucket.name}.${ + new URL(endpoint).hostname + }/${this.fileName}`; + return this.baseUrl; + } + /** * Initiates a multipart upload (MPU) to the XML API and stores the resultant upload id. * @@ -261,7 +272,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { */ async initiateUpload(headers?: {[key: string]: string}): Promise { const headersObject = new Headers(headers); - const url = `${this.baseUrl}?uploads`; + const url = `${await this.#getBaseUrl()}?uploads`; return AsyncRetry(async bail => { try { const res = await this.authClient.request< @@ -302,7 +313,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { chunk: Buffer, validation?: 'md5' | 'crc32c' | false, ): Promise { - const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`; + const url = `${await this.#getBaseUrl()}?partNumber=${partNumber}&uploadId=${this.uploadId}`; const headers: Headers = this.#setGoogApiClientHeaders(); if (validation === 'md5') { @@ -341,7 +352,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { * @returns {Promise} */ async completeUpload(): Promise { - const url = `${this.baseUrl}?uploadId=${this.uploadId}`; + const url = `${await this.#getBaseUrl()}?uploadId=${this.uploadId}`; const sortedMap = new Map( [...this.partsMap.entries()].sort((a, b) => a[0] - b[0]), ); @@ -380,7 +391,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { * @returns {Promise} */ async abortUpload(): Promise { - const url = `${this.baseUrl}?uploadId=${this.uploadId}`; + const url = `${await this.#getBaseUrl()}?uploadId=${this.uploadId}`; return AsyncRetry(async bail => { try { const res = await this.authClient.request( diff --git a/handwritten/storage/test/file.ts b/handwritten/storage/test/file.ts index 8f9e7d170fc..3bd09760e48 100644 --- a/handwritten/storage/test/file.ts +++ b/handwritten/storage/test/file.ts @@ -120,6 +120,9 @@ describe('File', () => { }); beforeEach(() => { + sandbox + .stub(STORAGE, 'getEndpoint') + .callsFake(async () => STORAGE['apiEndpoint']); BUCKET = new Bucket(STORAGE, 'bucket-name'); file = new File(BUCKET, FILE_NAME); @@ -3694,8 +3697,7 @@ describe('File', () => { STORAGE = new Storage({projectId: PROJECT_ID}); const customEndpoint = 'https://my-custom-endpoint.com'; - STORAGE.apiEndpoint = customEndpoint; - STORAGE.customEndpoint = true; + STORAGE['apiEndpoint'] = customEndpoint; CONFIG.virtualHostedStyle = true; CONFIG.bucketBoundHostname = 'http://domain.tld'; @@ -3707,6 +3709,57 @@ describe('File', () => { }); }); + it('should append bucket name to the URL when using the emulator', done => { + const emulatorHost = 'http://127.0.0.1:9199'; + const originalApiEndpoint = STORAGE['apiEndpoint']; + const originalAuthClient = STORAGE.storageTransport.authClient; + const originalEnvHost = process.env.STORAGE_EMULATOR_HOST; + + process.env.STORAGE_EMULATOR_HOST = emulatorHost; + STORAGE['apiEndpoint'] = emulatorHost; + STORAGE.storageTransport.authClient = + BUCKET.storage.storageTransport.authClient; + + file.generateSignedPostPolicyV4(CONFIG, (err, res) => { + STORAGE['apiEndpoint'] = originalApiEndpoint; + STORAGE.storageTransport.authClient = originalAuthClient; + if (originalEnvHost) { + process.env.STORAGE_EMULATOR_HOST = originalEnvHost; + } else { + delete process.env.STORAGE_EMULATOR_HOST; + } + + assert.ifError(err); + assert.strictEqual(res?.url, `${emulatorHost}/${file.bucket.name}`); + done(); + }); + }); + + it('should append bucket name when universeDomain is resolved dynamically via getEndpoint', done => { + const dynamicUniverse = 'my-universe.com'; + const dynamicEndpoint = `https://storage.${dynamicUniverse}`; + const originalAuthClient = STORAGE.storageTransport.authClient; + const originalUniverseDomain = STORAGE.universeDomain; + + STORAGE.storageTransport.authClient = { + ...BUCKET.storage.storageTransport.authClient, + getUniverseDomain: sandbox.stub().resolves(dynamicUniverse), + }; + (STORAGE.getEndpoint as sinon.SinonStub).callsFake(async () => { + STORAGE.universeDomain = dynamicUniverse; + return dynamicEndpoint; + }); + + file.generateSignedPostPolicyV4(CONFIG, (err, res) => { + STORAGE.storageTransport.authClient = originalAuthClient; + STORAGE.universeDomain = originalUniverseDomain; + + assert.ifError(err); + assert.strictEqual(res?.url, `${dynamicEndpoint}/${file.bucket.name}/`); + done(); + }); + }); + describe('expires', () => { it('should accept Date objects', () => { const expires = new Date(Date.now() + 1000 * 60); @@ -4190,10 +4243,10 @@ describe('File', () => { it('should correctly format URL and method in the request', done => { gaxiosStub.resolves({data: {}}); - // const expectedUrl = `https://${file.storage.apiEndpoint}/storage/v1/b/${BUCKET.name}/o/${encodeURIComponent(file.name)}`; - const baseUrl = file.storage.apiEndpoint.startsWith('http') - ? file.storage.apiEndpoint - : `https://${file.storage.apiEndpoint}`; + // const expectedUrl = `https://${file.storage['apiEndpoint']}/storage/v1/b/${BUCKET.name}/o/${encodeURIComponent(file.name)}`; + const baseUrl = file.storage['apiEndpoint'].startsWith('http') + ? file.storage['apiEndpoint'] + : `https://${file.storage['apiEndpoint']}`; const expectedUrl = `${baseUrl}/storage/v1/b/${BUCKET.name}/o/${encodeURIComponent(file.name)}`; @@ -5904,12 +5957,15 @@ describe('File', () => { }); describe('startSimpleUpload_', () => { - it('should get a writable stream', async done => { + it('should get a writable stream', async () => { + let makeRequestCalled = false; file.storageTransport.makeRequest = sandbox.stub().callsFake(() => { - done(); + makeRequestCalled = true; + return Promise.resolve({}); }); await file.startSimpleUpload_(duplexify()); + assert.strictEqual(makeRequestCalled, true); }); it('should pass the required arguments', async () => { @@ -6045,7 +6101,7 @@ describe('File', () => { it('should destroy the stream', done => { const stream = duplexify(); - file.startSimpleUpload_(stream); + void file.startSimpleUpload_(stream); stream.on('error', (err: Error) => { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -6066,22 +6122,22 @@ describe('File', () => { .resolves({data: body, resp}); }); - it('should set the metadata', () => { + it('should set the metadata', async () => { const stream = duplexify(); - file.startSimpleUpload_(stream); + await file.startSimpleUpload_(stream); assert.deepEqual(file.metadata, body); }); - it('should emit the response', () => { + it('should emit the response', async () => { const stream = duplexify(); stream.on('response', resp_ => { assert.strictEqual(resp_, resp); }); - file.startSimpleUpload_(stream); + await file.startSimpleUpload_(stream); }); it('should emit complete', async () => { diff --git a/handwritten/storage/test/index.ts b/handwritten/storage/test/index.ts index 03377001262..eae423f82a9 100644 --- a/handwritten/storage/test/index.ts +++ b/handwritten/storage/test/index.ts @@ -33,6 +33,7 @@ import { Storage, StorageExceptionMessages, } from '../src/storage.js'; +import {GoogleAuth, PassThroughClient} from 'google-auth-library'; import {StorageTransport} from '../src/storage-transport.js'; // eslint-disable-next-line @typescript-eslint/no-var-requires @@ -78,31 +79,8 @@ describe('Storage', () => { apiEndpoint, }); assert.strictEqual(storage.baseUrl, `${apiEndpoint}/storage/v1`); - assert.strictEqual(storage.apiEndpoint, `${apiEndpoint}`); + assert.strictEqual(storage['apiEndpoint'], `${apiEndpoint}`); }); - - it('should not set `customEndpoint` if `apiEndpoint` matches default', () => { - const apiEndpoint = 'https://storage.googleapis.com'; - const storage = new Storage({ - apiEndpoint, - }); - - assert.strictEqual(storage.apiEndpoint, apiEndpoint); - assert.strictEqual(storage.customEndpoint, false); - }); - - it('should not set `customEndpoint` if `apiEndpoint` matches default (w/ universe domain)', () => { - const universeDomain = 'my.universe'; - const apiEndpoint = `https://storage.${universeDomain}`; - const storage = new Storage({ - apiEndpoint, - universeDomain, - }); - - assert.strictEqual(storage.apiEndpoint, apiEndpoint); - assert.strictEqual(storage.customEndpoint, false); - }); - it('should propagate autoRetry in retryOptions', () => { const autoRetry = false; const storage = new Storage({ @@ -286,15 +264,6 @@ describe('Storage', () => { assert.strictEqual(storage.retryOptions.retryableErrorFn!(error), true); }); - it('should set customEndpoint to true when using apiEndpoint', () => { - const storage = new Storage({ - projectId: PROJECT_ID, - apiEndpoint: 'https://apiendpoint', - }); - - assert.strictEqual(storage.customEndpoint, true); - }); - it('should prepend apiEndpoint with default protocol', () => { const protocollessApiEndpoint = 'some.fake.endpoint'; const storage = new Storage({ @@ -306,7 +275,7 @@ describe('Storage', () => { `https://${protocollessApiEndpoint}/storage/v1`, ); assert.strictEqual( - storage.apiEndpoint, + storage['apiEndpoint'], `https://${protocollessApiEndpoint}`, ); }); @@ -318,7 +287,7 @@ describe('Storage', () => { apiEndpoint, }); assert.strictEqual(storage.baseUrl, `${apiEndpoint}storage/v1`); - assert.strictEqual(storage.apiEndpoint, 'https://some.fake.endpoint'); + assert.strictEqual(storage['apiEndpoint'], 'https://some.fake.endpoint'); }); it('should accept a `crc32cGenerator`', () => { @@ -345,12 +314,22 @@ describe('Storage', () => { ); }); - it('should accept and use a `universeDomain`', () => { + it('should accept and use a `universeDomain`', async () => { const universeDomain = 'my-universe.com'; + const stub = sinon + .stub(GoogleAuth.prototype, 'getUniverseDomain') + .resolves(universeDomain); - const storage = new Storage({universeDomain}); + try { + const storage = new Storage({universeDomain}); - assert.equal(storage.apiEndpoint, `https://storage.${universeDomain}`); + assert.equal( + await storage.getEndpoint(), + `https://storage.${universeDomain}`, + ); + } finally { + stub.restore(); + } }); describe('STORAGE_EMULATOR_HOST', () => { @@ -371,7 +350,7 @@ describe('Storage', () => { assert.strictEqual(storage.baseUrl, EMULATOR_HOST); assert.strictEqual( - storage.apiEndpoint, + storage['apiEndpoint'], 'https://internal.benchmark.com/path', ); }); @@ -383,7 +362,7 @@ describe('Storage', () => { }); assert.strictEqual(storage.baseUrl, EMULATOR_HOST); - assert.strictEqual(storage.apiEndpoint, 'https://some.api.com'); + assert.strictEqual(storage['apiEndpoint'], 'https://some.api.com'); }); it('should prepend default protocol and strip trailing slash', () => { @@ -396,17 +375,40 @@ describe('Storage', () => { assert.strictEqual(storage.baseUrl, EMULATOR_HOST); assert.strictEqual( - storage.apiEndpoint, + storage['apiEndpoint'], 'https://internal.benchmark.com/path', ); }); - it('should set customEndpoint to true', () => { + it('should use PassThroughClient when authClient is not provided', () => { const storage = new Storage({ projectId: PROJECT_ID, }); - assert.strictEqual(storage.customEndpoint, true); + assert( + storage.storageTransport.authClient.cachedCredential instanceof + PassThroughClient, + ); + }); + + it('should not use PassThroughClient when keyFilename or credentials are provided', () => { + const storageWithKeyFile = new Storage({ + projectId: PROJECT_ID, + keyFilename: '/path/to/keyfile.json', + }); + assert.strictEqual( + storageWithKeyFile.storageTransport.authClient.cachedCredential, + null, + ); + + const storageWithCreds = new Storage({ + projectId: PROJECT_ID, + credentials: {client_email: 'a@b.com', private_key: 'key'}, + }); + assert.strictEqual( + storageWithCreds.storageTransport.authClient.cachedCredential, + null, + ); }); }); }); diff --git a/handwritten/storage/test/resumable-upload.ts b/handwritten/storage/test/resumable-upload.ts index 1661fd1012b..9d80a0794d5 100644 --- a/handwritten/storage/test/resumable-upload.ts +++ b/handwritten/storage/test/resumable-upload.ts @@ -2708,7 +2708,7 @@ describe('resumable-upload', () => { assert.deepStrictEqual(headersObj, {}); }); - it('should bypass authentication if emulator context detected', async () => { + it('should use authentication with custom endpoint', async () => { up = upload({ bucket: BUCKET, file: FILE, @@ -2724,115 +2724,21 @@ describe('resumable-upload', () => { retryOptions: RETRY_OPTIONS, }); const scopes = [ - nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}), - ]; - const res = await up.makeRequest(REQ_OPTS); - scopes.forEach(x => x.done()); - assert.strictEqual( - res.config.url!.toString(), - REQ_OPTS.url + queryPath.slice(1), - ); - assert.ok(res.headers); - }); - - it('should use authentication with custom endpoint when useAuthWithCustomEndpoint is true', async () => { - up = upload({ - bucket: BUCKET, - file: FILE, - customRequestOptions: CUSTOM_REQUEST_OPTIONS, - generation: GENERATION, - metadata: METADATA, - origin: ORIGIN, - params: PARAMS, - predefinedAcl: PREDEFINED_ACL, - userProject: USER_PROJECT, - authConfig: {keyFile}, - apiEndpoint: 'https://custom-proxy.example.com', - useAuthWithCustomEndpoint: true, - retryOptions: RETRY_OPTIONS, - }); - - // Mock the authorization request - mockAuthorizeRequest(); - - // Mock the actual request with auth header expectation - const scopes = [ + mockAuthorizeRequest(), nock(REQ_OPTS.url!) .matchHeader('authorization', /Bearer .+/) .get(queryPath) .reply(200, undefined, {}), ]; - const res = await up.makeRequest(REQ_OPTS); scopes.forEach(x => x.done()); assert.strictEqual( res.config.url!.toString(), REQ_OPTS.url + queryPath.slice(1), ); - // Headers should include authorization assert.ok(getHeader(res.config.headers, 'Authorization')); }); - it('should bypass authentication with custom endpoint when useAuthWithCustomEndpoint is false', async () => { - up = upload({ - bucket: BUCKET, - file: FILE, - customRequestOptions: CUSTOM_REQUEST_OPTIONS, - generation: GENERATION, - metadata: METADATA, - origin: ORIGIN, - params: PARAMS, - predefinedAcl: PREDEFINED_ACL, - userProject: USER_PROJECT, - authConfig: {keyFile}, - apiEndpoint: 'https://storage-emulator.local', - useAuthWithCustomEndpoint: false, - retryOptions: RETRY_OPTIONS, - }); - - const scopes = [ - nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}), - ]; - const res = await up.makeRequest(REQ_OPTS); - scopes.forEach(x => x.done()); - assert.strictEqual( - res.config.url!.toString(), - REQ_OPTS.url + queryPath.slice(1), - ); - // When auth is bypassed, no auth headers should be present - assert.ok(res.headers); - }); - - it('should bypass authentication with custom endpoint when useAuthWithCustomEndpoint is undefined (backward compatibility)', async () => { - up = upload({ - bucket: BUCKET, - file: FILE, - customRequestOptions: CUSTOM_REQUEST_OPTIONS, - generation: GENERATION, - metadata: METADATA, - origin: ORIGIN, - params: PARAMS, - predefinedAcl: PREDEFINED_ACL, - userProject: USER_PROJECT, - authConfig: {keyFile}, - apiEndpoint: 'https://storage-emulator.local', - // useAuthWithCustomEndpoint is intentionally not set - retryOptions: RETRY_OPTIONS, - }); - - const scopes = [ - nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}), - ]; - const res = await up.makeRequest(REQ_OPTS); - scopes.forEach(x => x.done()); - assert.strictEqual( - res.config.url!.toString(), - REQ_OPTS.url + queryPath.slice(1), - ); - // When auth is bypassed (backward compatibility), no auth headers should be present - assert.ok(res.headers); - }); - it('should combine customRequestOptions', done => { const up = upload({ bucket: BUCKET, diff --git a/handwritten/storage/test/signer.ts b/handwritten/storage/test/signer.ts index 7432cf19359..98fa681e322 100644 --- a/handwritten/storage/test/signer.ts +++ b/handwritten/storage/test/signer.ts @@ -64,6 +64,9 @@ describe('signer', () => { afterEach(() => fakeTimers.restore()); beforeEach(() => { + sandbox + .stub(GoogleAuth.prototype, 'getUniverseDomain') + .resolves('googleapis.com'); authClient = { sign: async () => 'signature', getCredentials: async () => ({client_email: CLIENT_EMAIL}), @@ -447,6 +450,32 @@ describe('signer', () => { const signedUrl = await signer.getSignedUrl(CONFIG); assert(signedUrl.includes(qsStringify(query))); }); + + it('should call storage.getEndpoint only once for v4 signed URLs when host and cname are not set', async () => { + const getEndpointSpy = sandbox.spy(storage, 'getEndpoint'); + CONFIG.version = 'v4'; + + await signer.getSignedUrl(CONFIG); + assert.strictEqual(getEndpointSpy.callCount, 1); + }); + + it('should not call storage.getEndpoint when host or cname is provided', async () => { + const getEndpointSpy = sandbox.spy(storage, 'getEndpoint'); + + await signer.getSignedUrl({ + ...CONFIG, + version: 'v4', + host: 'https://custom.host.com', + }); + assert.strictEqual(getEndpointSpy.callCount, 0); + + await signer.getSignedUrl({ + ...CONFIG, + version: 'v4', + cname: 'https://cdn.example.com', + }); + assert.strictEqual(getEndpointSpy.callCount, 0); + }); }); describe('getSignedUrlV2', () => {