refactor(storage): revamp apiEndpoint handling, enforce uniform authentication, and use PassThroughClient for emulator

handwritten/storage/src/file.ts

@@ -2027,31 +2027,41 @@ class File extends ServiceObject<File, FileMetadata> {
       retryOptions.autoRetry = false;
     }
 
-    resumableUpload.createURI(
-      {
-        authClient: this.storage.storageTransport.authClient,
-        apiEndpoint: this.storage.apiEndpoint,
-        bucket: this.bucket.name,
-        file: this.name,
-        generation: this.generation,
-        key: this.encryptionKey === null ? undefined : this.encryptionKey,
-        kmsKeyName: this.kmsKeyName,
-        metadata: options.metadata,
-        offset: options.offset,
-        origin: options.origin,
-        predefinedAcl: options.predefinedAcl,
-        private: options.private,
-        public: options.public,
-        userProject: options.userProject || this.userProject,
-        retryOptions: retryOptions,
-        params: options?.preconditionOpts || this.instancePreconditionOpts,
-        universeDomain: this.bucket.storage.universeDomain,
-        useAuthWithCustomEndpoint: this.storage.useAuthWithCustomEndpoint,
-        [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
-      },
-      callback!,
-    );
-    this.storage.retryOptions.autoRetry = this.instanceRetryValue;
+    void (async () => {
+      try {
+        const apiEndpoint =
+          (typeof this.storage.getEndpoint === 'function'
+            ? await this.storage.getEndpoint()
+            : undefined) || this.storage['apiEndpoint'];
+        resumableUpload.createURI(
+          {
+            authClient: this.storage.storageTransport.authClient,
+            apiEndpoint,
+            bucket: this.bucket.name,
+            file: this.name,
+            generation: this.generation,
+            key: this.encryptionKey === null ? undefined : this.encryptionKey,
+            kmsKeyName: this.kmsKeyName,
+            metadata: options.metadata,
+            offset: options.offset,
+            origin: options.origin,
+            predefinedAcl: options.predefinedAcl,
+            private: options.private,
+            public: options.public,
+            userProject: options.userProject || this.userProject,
+            retryOptions: {...retryOptions},
+            params: options?.preconditionOpts || this.instancePreconditionOpts,
+            universeDomain: this.bucket.storage.universeDomain,
+            [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
+          },
+          callback!,
+        );
+      } catch (err) {
+        callback!(err as Error);
+      } finally {
+        this.storage.retryOptions.autoRetry = this.instanceRetryValue;
+      }
+    })();
   }
 
   /**
@@ -2437,7 +2447,7 @@ class File extends ServiceObject<File, FileMetadata> {
         );
 
         if (options.resumable === false) {
-          this.startSimpleUpload_(
+          void this.startSimpleUpload_(
             fileWriteStream,
             options as CreateWriteStreamOptionsInternal,
           );
@@ -3272,14 +3282,26 @@ class File extends ServiceObject<File, FileMetadata> {
           options.signingEndpoint,
         );
         const signatureHex = Buffer.from(signature, 'base64').toString('hex');
-        const universe = this.parent.storage.universeDomain;
         fields['policy'] = policyBase64;
         fields['x-goog-signature'] = signatureHex;
 
         let url: string;
 
-        if (this.storage.customEndpoint) {
-          url = this.storage.apiEndpoint;
+        const EMULATOR_HOST = process.env.STORAGE_EMULATOR_HOST;
+        const endpoint =
+          (typeof this.storage.getEndpoint === 'function'
+            ? await this.storage.getEndpoint()
+            : undefined) || this.storage['apiEndpoint'];
+        const universe =
+          this.parent.storage.universeDomain ||
+          this.storage.universeDomain ||
+          'googleapis.com';
+        const defaultEndpoint = `https://storage.${universe}`;
+
+        if (endpoint !== defaultEndpoint && typeof EMULATOR_HOST === 'string') {
+          url = `${endpoint}/${this.bucket.name}`;
+        } else if (endpoint !== defaultEndpoint) {
+          url = endpoint;
         } else if (options.virtualHostedStyle) {
           url = `https://${this.bucket.name}.storage.${universe}/`;
         } else if (options.bucketBoundHostname) {
@@ -3610,11 +3632,6 @@ class File extends ServiceObject<File, FileMetadata> {
       undefined,
       callback,
     );
-    const baseUrl = this.storage.apiEndpoint.startsWith('http')
-      ? this.storage.apiEndpoint
-      : `https://${this.storage.apiEndpoint}`;
-
-    const url = `${baseUrl}/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`;
 
     const gaxios = new Gaxios();
     const storageInterceptors = this.storage?.interceptors || [];
@@ -3627,6 +3644,16 @@ class File extends ServiceObject<File, FileMetadata> {
     // eslint-disable-next-line @typescript-eslint/no-floating-promises
     (async () => {
       try {
+        const endpoint =
+          (typeof this.storage.getEndpoint === 'function'
+            ? await this.storage.getEndpoint()
+            : undefined) || this.storage['apiEndpoint'];
+        const baseUrl = endpoint.startsWith('http')
+          ? endpoint
+          : `https://${endpoint}`;
+
+        const url = `${baseUrl}/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`;
+
         await gaxios.request({
           method: 'GET',
           url,
@@ -3823,7 +3850,7 @@ class File extends ServiceObject<File, FileMetadata> {
    * ```
    */
   publicUrl(): string {
-    return `${this.storage.apiEndpoint}/${
+    return `${this.storage['apiEndpoint']}/${
       this.bucket.name
     }/${encodeURIComponent(this.name)}`;
   }
@@ -4755,59 +4782,64 @@ class File extends ServiceObject<File, FileMetadata> {
     ) {
       retryOptions.autoRetry = false;
     }
-    const cfg = {
-      authClient: this.storage.storageTransport.authClient,
-      apiEndpoint: this.storage.apiEndpoint,
-      bucket: this.bucket.name,
-      file: this.name,
-      generation: this.generation,
-      isPartialUpload: options.isPartialUpload,
-      key: this.encryptionKey === null ? undefined : this.encryptionKey,
-      kmsKeyName: this.kmsKeyName,
-      metadata: options.metadata,
-      offset: options.offset,
-      predefinedAcl: options.predefinedAcl,
-      private: options.private,
-      public: options.public,
-      uri: options.uri,
-      userProject: options.userProject || this.userProject,
-      retryOptions: {...retryOptions},
-      params: options?.preconditionOpts || this.instancePreconditionOpts,
-      chunkSize: options?.chunkSize,
-      highWaterMark: options?.highWaterMark,
-      universeDomain: this.bucket.storage.universeDomain,
-      useAuthWithCustomEndpoint: this.storage.useAuthWithCustomEndpoint,
-      [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
-    };
+    const uploadRetryOptions = {...retryOptions};
+    this.storage.retryOptions.autoRetry = this.instanceRetryValue;
 
-    let uploadStream: resumableUpload.Upload;
+    void (async () => {
+      let uploadStream: resumableUpload.Upload;
 
-    try {
-      uploadStream = resumableUpload.upload(cfg);
-    } catch (error) {
-      dup.destroy(error as Error);
-      this.storage.retryOptions.autoRetry = this.instanceRetryValue;
-      return;
-    }
+      try {
+        const apiEndpoint =
+          (typeof this.storage.getEndpoint === 'function'
+            ? await this.storage.getEndpoint()
+            : undefined) || this.storage['apiEndpoint'];
+        const cfg = {
+          authClient: this.storage.storageTransport.authClient,
+          apiEndpoint,
+          bucket: this.bucket.name,
+          file: this.name,
+          generation: this.generation,
+          isPartialUpload: options.isPartialUpload,
+          key: this.encryptionKey === null ? undefined : this.encryptionKey,
+          kmsKeyName: this.kmsKeyName,
+          metadata: options.metadata,
+          offset: options.offset,
+          predefinedAcl: options.predefinedAcl,
+          private: options.private,
+          public: options.public,
+          uri: options.uri,
+          userProject: options.userProject || this.userProject,
+          retryOptions: uploadRetryOptions,
+          params: options?.preconditionOpts || this.instancePreconditionOpts,
+          chunkSize: options?.chunkSize,
+          highWaterMark: options?.highWaterMark,
+          universeDomain: this.bucket.storage.universeDomain,
+          [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
+        };
+        uploadStream = resumableUpload.upload(cfg);
+      } catch (error) {
+        dup.destroy(error as Error);
+        return;
+      }
 
-    uploadStream
-      .on('response', resp => {
-        dup.emit('response', resp);
-      })
-      .on('uri', uri => {
-        dup.emit('uri', uri);
-      })
-      .on('metadata', metadata => {
-        this.metadata = metadata;
-        dup.emit('metadata');
-      })
-      .on('finish', () => {
-        dup.emit('complete');
-      })
-      .on('progress', evt => dup.emit('progress', evt));
+      uploadStream
+        .on('response', resp => {
+          dup.emit('response', resp);
+        })
+        .on('uri', uri => {
+          dup.emit('uri', uri);
+        })
+        .on('metadata', metadata => {
+          this.metadata = metadata;
+          dup.emit('metadata');
+        })
+        .on('finish', () => {
+          dup.emit('complete');
+        })
+        .on('progress', evt => dup.emit('progress', evt));
 
-    dup.setWritable(uploadStream);
-    this.storage.retryOptions.autoRetry = this.instanceRetryValue;
+      dup.setWritable(uploadStream);
+    })();
   }
 
   /**
@@ -4820,13 +4852,26 @@ class File extends ServiceObject<File, FileMetadata> {
    *
    * @private
    */
-  startSimpleUpload_(
+  async startSimpleUpload_(
     dup: Duplexify,
     options: CreateWriteStreamOptionsInternal = {},
-  ): void {
+  ): Promise<void> {
     options.metadata ??= {};
 
-    const apiEndpoint = this.storage.apiEndpoint;
+    const writeStream = new ProgressStream();
+    writeStream.on('progress', evt => dup.emit('progress', evt));
+    dup.setWritable(writeStream);
+
+    let apiEndpoint: string;
+    try {
+      apiEndpoint =
+        (typeof this.storage.getEndpoint === 'function'
+          ? await this.storage.getEndpoint()
+          : undefined) || this.storage['apiEndpoint'];
+    } catch (err) {
+      dup.destroy(err as Error);
+      return;
+    }
     const bucketName = this.bucket.name;
     const url = `${apiEndpoint}/upload/storage/v1/b/${bucketName}/o`;
 
@@ -4873,10 +4918,6 @@ class File extends ServiceObject<File, FileMetadata> {
       options.preconditionOpts,
     );
 
-    const writeStream = new ProgressStream();
-    writeStream.on('progress', evt => dup.emit('progress', evt));
-    dup.setWritable(writeStream);
-
     reqOpts.multipart = [
       {
         headers: new Headers({'Content-Type': 'application/json'}),
@@ -4914,7 +4955,7 @@ class File extends ServiceObject<File, FileMetadata> {
         dup.emit('response', resp);
         dup.emit('complete');
       })
-      .catch(err => dup.destroy(err));
+      ?.catch(err => dup.destroy(err));
   }
 
   disableAutoRetryConditionallyIdempotent_(

handwritten/storage/src/nodejs-common/util.ts

@@ -104,11 +104,13 @@ export interface MakeAuthenticatedRequestFactoryConfig extends Omit<
 
   /**
    * If true, just return the provided request options. Default: false.
+   * @deprecated
    */
   customEndpoint?: boolean;
 
   /**
    * If true, will authenticate when using a custom endpoint. Default: false.
+   * @deprecated
    */
   useAuthWithCustomEndpoint?: boolean;
 

handwritten/storage/src/resumable-upload.ts

@@ -18,7 +18,6 @@ import {
   GaxiosResponse,
   GaxiosError,
 } from 'gaxios';
-import * as gaxios from 'gaxios';
 import {
   DEFAULT_UNIVERSE,
   GoogleAuth,
@@ -247,6 +246,7 @@ export interface UploadConfig extends Pick<WritableOptions, 'highWaterMark'> {
 
   /**
    * Controls whether or not to use authentication when using a custom endpoint.
+   * @deprecated
    */
   useAuthWithCustomEndpoint?: boolean;
 
@@ -385,33 +385,6 @@ export class Upload extends Writable {
     this.apiEndpoint = `https://storage.${universe}`;
     if (cfg.apiEndpoint && cfg.apiEndpoint !== this.apiEndpoint) {
       this.apiEndpoint = this.sanitizeEndpoint(cfg.apiEndpoint);
-
-      const hostname = new URL(this.apiEndpoint).hostname;
-
-      // check if it is a domain of a known universe
-      const isDomain = hostname === universe;
-      const isDefaultUniverseDomain = hostname === DEFAULT_UNIVERSE;
-
-      // check if it is a subdomain of a known universe
-      // by checking a last (universe's length + 1) of a hostname
-      const isSubDomainOfUniverse =
-        hostname.slice(-(universe.length + 1)) === `.${universe}`;
-      const isSubDomainOfDefaultUniverse =
-        hostname.slice(-(DEFAULT_UNIVERSE.length + 1)) ===
-        `.${DEFAULT_UNIVERSE}`;
-
-      if (
-        !isDomain &&
-        !isDefaultUniverseDomain &&
-        !isSubDomainOfUniverse &&
-        !isSubDomainOfDefaultUniverse
-      ) {
-        // a custom, non-universe domain,
-        // use gaxios only if useAuthWithCustomEndpoint is not true
-        if (cfg.useAuthWithCustomEndpoint !== true) {
-          this.authClient = gaxios;
-        }
-      }
     }
 
     this.baseURI = `${this.apiEndpoint}/upload/storage/v1/b`;