From 5d00c35b09d8a9d1cc538919169f1e4422378ff8 Mon Sep 17 00:00:00 2001 From: Alka Trivedi Date: Wed, 14 Oct 2026 11:42:44 +0000 Subject: [PATCH 1/3] fix(Spanner): retry and resume BatchWrite streams on UNAVAILABLE --- .../spanner/observability-test/database.ts | 60 ++ handwritten/spanner/src/batch-write-stream.ts | 390 +++++++++++++ handwritten/spanner/src/database.ts | 158 +++++- handwritten/spanner/src/index.ts | 17 +- .../spanner/src/metrics/metrics-tracer.ts | 26 +- .../spanner/src/request-stream-coordinator.ts | 23 +- .../spanner/test/batch-write-stream.ts | 534 ++++++++++++++++++ handwritten/spanner/test/database.ts | 217 ++++++- .../spanner/test/metrics/metrics-tracer.ts | 38 ++ .../spanner/test/mockserver/mockspanner.ts | 45 +- .../test/request-stream-coordinator.ts | 230 ++++++++ handwritten/spanner/test/spanner.ts | 104 ++++ 12 files changed, 1792 insertions(+), 50 deletions(-) create mode 100644 handwritten/spanner/src/batch-write-stream.ts create mode 100644 handwritten/spanner/test/batch-write-stream.ts diff --git a/handwritten/spanner/observability-test/database.ts b/handwritten/spanner/observability-test/database.ts index 1b3a3e23df4b..83c9b008f31d 100644 --- a/handwritten/spanner/observability-test/database.ts +++ b/handwritten/spanner/observability-test/database.ts @@ -1285,6 +1285,66 @@ describe('Database', () => { fakeDataStream.emit('data', 'response'); fakeDataStream.end('end'); }); + + it('ends span and does not start RPC when stream is destroyed before getSession resolves', done => { + let sessionCallback: + ((err: Error | null, session?: FakeSession) => void) | null = null; + getSessionStub.callsFake(callback => { + sessionCallback = callback; + }); + + const stream = database.batchWriteAtLeastOnce(mutationGroups, options); + stream.destroy(); + + setImmediate(() => { + sessionCallback!(null, fakeSession); + const spans = traceExporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1, 'Exactly 1 span expected'); + withAllSpansHaveDBName(spans); + assert.strictEqual( + spans[0].name, + 'CloudSpanner.Database.batchWriteAtLeastOnce', + ); + assert.strictEqual(spans[0].events.length, 0); + done(); + }); + }); + + it('records error on span when stream is destroyed with an error', done => { + const abortErr = new Error('abort'); + const stream = database.batchWriteAtLeastOnce(mutationGroups, options); + stream.on('error', err => { + assert.strictEqual(err, abortErr); + const spans = traceExporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1, 'Exactly 1 span expected'); + withAllSpansHaveDBName(spans); + assert.strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + assert.strictEqual(spans[0].status.message, 'abort'); + done(); + }); + stream.destroy(abortErr); + }); + + it('ends span immediately with no events when mutationGroups is empty', done => { + database + .batchWriteAtLeastOnce([], options) + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', done) + .on('end', () => { + const spans = traceExporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1, 'Exactly 1 span expected'); + withAllSpansHaveDBName(spans); + assert.strictEqual( + spans[0].name, + 'CloudSpanner.Database.batchWriteAtLeastOnce', + ); + assert.strictEqual(spans[0].events.length, 0); + assert.notStrictEqual(spans[0].status.code, SpanStatusCode.ERROR); + done(); + }); + }); }); describe('runTransaction', () => { diff --git a/handwritten/spanner/src/batch-write-stream.ts b/handwritten/spanner/src/batch-write-stream.ts new file mode 100644 index 000000000000..c20144828840 --- /dev/null +++ b/handwritten/spanner/src/batch-write-stream.ts @@ -0,0 +1,390 @@ +/*! + * Copyright 2026 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {CallOptions, grpc} from 'google-gax'; +import {Readable, Transform} from 'stream'; +import * as through from 'through2'; +import {protos} from '@google-cloud/spanner-api'; +import {DeadlineError, isRetryableInternalError} from './transaction-runner'; +import {getRetryInfo} from './protos'; + +const RETRY_INFO_BIN = 'google.rpc.retryinfo-bin'; +const RetryInfo = getRetryInfo(); + +type IMutationGroup = protos.google.spanner.v1.BatchWriteRequest.IMutationGroup; +type IBatchWriteResponse = protos.google.spanner.v1.IBatchWriteResponse; + +export interface BatchWriteStreamOptions { + gaxOptions?: CallOptions; + onTerminate?: (finalStatus: grpc.status) => void; +} + +// Default BatchWrite retry/backoff parameters from spanner_client_config.json +const DEFAULT_INITIAL_RETRY_DELAY_MS = 100; +const DEFAULT_MAX_RETRY_DELAY_MS = 60000; +const DEFAULT_RETRY_DELAY_MULTIPLIER = 1.3; +const DEFAULT_TOTAL_TIMEOUT_MS = 3600000; // 1 hour + +/** + * Extracts server-supplied retry delay (in ms) from `google.rpc.retryinfo-bin` + * trailing metadata if present, otherwise computes jittered exponential backoff. + */ +export function getNextRetryDelayMs( + err: grpc.ServiceError, + retryAttempt: number, + backoffSettings?: Partial< + NonNullable['backoffSettings']> + >, +): number { + const retryInfoValues = err?.metadata && err.metadata.get(RETRY_INFO_BIN); + if (retryInfoValues && retryInfoValues.length > 0) { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const {retryDelay} = (RetryInfo as any).decode(retryInfoValues[0]); + if (retryDelay) { + let {seconds} = retryDelay; + if ( + seconds && + typeof seconds !== 'number' && + typeof seconds.toNumber === 'function' + ) { + seconds = seconds.toNumber(); + } + const secondsInMs = Math.floor(Number(seconds || 0)) * 1000; + const nanosInMs = Math.floor(Number(retryDelay.nanos || 0)) / 1e6; + const delayMs = secondsInMs + nanosInMs; + if (delayMs >= 0) { + return delayMs; + } + } + } catch { + // Fall back to exponential backoff if RetryInfo decoding fails. + } + } + + const initialDelay = + backoffSettings?.initialRetryDelayMillis ?? DEFAULT_INITIAL_RETRY_DELAY_MS; + const maxDelay = + backoffSettings?.maxRetryDelayMillis ?? DEFAULT_MAX_RETRY_DELAY_MS; + const multiplier = + backoffSettings?.retryDelayMultiplier ?? DEFAULT_RETRY_DELAY_MULTIPLIER; + + const unclampedDelay = + initialDelay * Math.pow(multiplier, Math.max(0, retryAttempt - 1)); + const cappedDelay = Math.min(maxDelay, unclampedDelay); + + // Jitter in [0.8 * cappedDelay, 1.2 * cappedDelay] + const jitterFactor = 0.8 + Math.random() * 0.4; + return Math.floor(cappedDelay * jitterFactor); +} + +/** + * Creates a readable object stream that executes `BatchWrite` and transparently + * retries and resumes on transient stream failures (`UNAVAILABLE`, retryable `INTERNAL`, + * and premature stream `end`) for only the unresolved mutation groups. + */ +export function batchWriteStream( + mutationGroups: IMutationGroup[], + makeRequest: ( + currentMutationGroups: IMutationGroup[], + remainingTimeoutMs: number, + ) => Readable, + options: BatchWriteStreamOptions = {}, +): Transform { + const gaxOptions = options.gaxOptions; + const backoffSettings = gaxOptions?.retry?.backoffSettings; + const retryableCodes = gaxOptions?.retry?.retryCodes ?? [ + grpc.status.UNAVAILABLE, + ]; + const timeout = + gaxOptions?.timeout ?? + backoffSettings?.totalTimeoutMillis ?? + DEFAULT_TOTAL_TIMEOUT_MS; + const maxRetries = backoffSettings?.maxRetries ?? Infinity; + + // Track pending mutation groups by their original caller index [0..N-1]. + // Resolved entries are set to `null` so large mutation payloads can be GC'd. + const totalGroups = mutationGroups ? mutationGroups.length : 0; + const pending: Array = mutationGroups + ? [...mutationGroups] + : []; + let remaining = totalGroups; + let sawNonObjectChunk = false; + + // Maps current RPC request index -> original caller index. + let streamIndexes: number[] = []; + + let activeRequestStream: Readable | null = null; + let retryTimer: NodeJS.Timeout | null = null; + let retryCount = 0; + const startTime = Date.now(); + let terminated = false; + + const userStream: Transform = through.obj( + (chunk: IBatchWriteResponse, _enc, next) => { + next(null, chunk); + }, + ); + + const finalize = (status: grpc.status) => { + if (terminated) { + return; + } + terminated = true; + if (retryTimer) { + clearTimeout(retryTimer); + retryTimer = null; + } + options.onTerminate?.(status); + }; + + const cleanupActiveRequestStream = () => { + if (!activeRequestStream) { + return; + } + const streamToClean = activeRequestStream; + activeRequestStream = null; + streamToClean.removeAllListeners('data'); + streamToClean.removeAllListeners('end'); + streamToClean.removeAllListeners('error'); + streamToClean.on('error', () => {}); + if (!streamToClean.destroyed) { + streamToClean.destroy(); + } + }; + + // Intercept stream destruction and close so early cancellation by the caller + // immediately clears any pending retry timer and aborts the active RPC stream. + const originalDestroy = userStream._destroy; + userStream._destroy = function ( + error: Error | null, + callback: (error?: Error | null) => void, + ) { + if (!terminated) { + cleanupActiveRequestStream(); + finalize( + (error as grpc.ServiceError | null)?.code ?? grpc.status.CANCELLED, + ); + } + if (typeof originalDestroy === 'function') { + originalDestroy.call(this, error, callback); + } else { + callback(error); + } + }; + + userStream.once('close', () => { + if (!terminated) { + cleanupActiveRequestStream(); + finalize(grpc.status.CANCELLED); + } + }); + + const isRetryableError = (err: grpc.ServiceError): boolean => { + if (!err || typeof err.code !== 'number') { + return false; + } + return ( + retryableCodes.includes(err.code) || + (retryableCodes.includes(grpc.status.UNAVAILABLE) && + isRetryableInternalError(err)) + ); + }; + + const handleRetryOrFail = (err: grpc.ServiceError) => { + cleanupActiveRequestStream(); + if (terminated || userStream.destroyed) { + return; + } + + const elapsed = Date.now() - startTime; + if ( + remaining > 0 && + retryCount < maxRetries && + elapsed < timeout && + isRetryableError(err) + ) { + retryCount++; + const delayMs = getNextRetryDelayMs(err, retryCount, backoffSettings); + if (elapsed + delayMs >= timeout) { + const deadlineErr = new DeadlineError(err); + finalize(grpc.status.DEADLINE_EXCEEDED); + userStream.destroy(deadlineErr); + return; + } + retryTimer = setTimeout(() => { + retryTimer = null; + if (!terminated && !userStream.destroyed) { + startAttempt(); + } + }, delayMs); + return; + } + + const finalError = + elapsed >= timeout && isRetryableError(err) + ? new DeadlineError(err) + : err; + finalize(finalError.code ?? grpc.status.UNKNOWN); + userStream.destroy(finalError); + }; + + const startAttempt = () => { + if (terminated || userStream.destroyed) { + return; + } + + if (remaining === 0) { + finalize(grpc.status.OK); + userStream.end(); + return; + } + + // Build the subset of unacknowledged MutationGroups and their index map. + const currentGroups: IMutationGroup[] = []; + streamIndexes = []; + for (let origIdx = 0; origIdx < totalGroups; origIdx++) { + const group = pending[origIdx]; + if (group !== null) { + currentGroups.push(group); + streamIndexes.push(origIdx); + } + } + + const elapsed = Date.now() - startTime; + const remainingTimeoutMs = Number.isFinite(timeout) + ? Math.max(0, timeout - elapsed) + : timeout; + const currentStream = makeRequest(currentGroups, remainingTimeoutMs); + activeRequestStream = currentStream; + + const onData = (response: IBatchWriteResponse) => { + if (terminated || userStream.destroyed) { + return; + } + + if (typeof response !== 'object' || response === null) { + sawNonObjectChunk = true; + if (!userStream.write(response)) { + currentStream.pause(); + } + return; + } + + const respIndexes = response.indexes; + if (Array.isArray(respIndexes)) { + // Validate all response indexes before mutating state. + for (let i = 0; i < respIndexes.length; i++) { + const idx = Number(respIndexes[i]); + if ( + !Number.isInteger(idx) || + idx < 0 || + idx >= streamIndexes.length + ) { + cleanupActiveRequestStream(); + const msg = `BatchWrite index ${idx} out of bounds for current request size ${streamIndexes.length}`; + const internalErr = Object.assign(new Error(msg), { + code: grpc.status.INTERNAL, + details: msg, + metadata: new grpc.Metadata(), + } as grpc.ServiceError); + finalize(grpc.status.INTERNAL); + userStream.destroy(internalErr); + return; + } + } + + // Translate retry-local indexes back to original caller indexes in-place + // and mark each acknowledged group as resolved (nulling reference for GC). + for (let i = 0; i < respIndexes.length; i++) { + const retryIdx = Number(respIndexes[i]); + const origIdx = streamIndexes[retryIdx]; + respIndexes[i] = origIdx; + if (pending[origIdx] !== null) { + pending[origIdx] = null; + remaining--; + } + } + } + + // Forward the response with translated indexes and respect downstream backpressure. + if (!userStream.write(response)) { + currentStream.pause(); + } + }; + + const onError = (err: grpc.ServiceError) => { + currentStream.removeListener('data', onData); + currentStream.removeListener('end', onEnd); + if (terminated || userStream.destroyed) { + return; + } + if (totalGroups > 0 && remaining === 0 && !sawNonObjectChunk) { + cleanupActiveRequestStream(); + finalize(grpc.status.OK); + userStream.end(); + return; + } + handleRetryOrFail(err); + }; + + const onEnd = () => { + currentStream.removeListener('data', onData); + currentStream.removeListener('error', onError); + activeRequestStream = null; + if (terminated || userStream.destroyed) { + return; + } + + // Premature stream end (EOF while remaining > 0) is treated as a + // retryable UNAVAILABLE error so remaining groups are retried. + if (remaining > 0 && !sawNonObjectChunk) { + const msg = `BatchWrite stream ended prematurely with ${remaining} unacknowledged mutation group(s)`; + const prematureEofErr = Object.assign(new Error(msg), { + code: grpc.status.UNAVAILABLE, + details: msg, + metadata: new grpc.Metadata(), + } as grpc.ServiceError); + handleRetryOrFail(prematureEofErr); + return; + } + + finalize(grpc.status.OK); + userStream.end(); + }; + + currentStream.on('data', onData); + currentStream.once('error', onError); + currentStream.once('end', onEnd); + }; + + // Propagate downstream drain to resume the active gRPC stream. + userStream.on('drain', () => { + activeRequestStream?.resume(); + }); + + if (totalGroups === 0) { + finalize(grpc.status.OK); + setImmediate(() => { + userStream.end(); + }); + return userStream; + } + + startAttempt(); + + return userStream; +} diff --git a/handwritten/spanner/src/database.ts b/handwritten/spanner/src/database.ts index a239358bc959..a5b5cf116554 100644 --- a/handwritten/spanner/src/database.ts +++ b/handwritten/spanner/src/database.ts @@ -38,6 +38,7 @@ import { } from 'google-gax'; import {Backup} from './backup'; import {BatchTransaction, TransactionIdentifier} from './batch-transaction'; +import {batchWriteStream} from './batch-write-stream'; import { GetSessionCallback, SessionFactory, @@ -3594,6 +3595,13 @@ class Database extends common.GrpcServiceObject { * rows being added to the mutation's table. We recommend structuring your mutation groups to * be idempotent to avoid this issue. * + * If the stream is interrupted by a retryable error (such as `UNAVAILABLE`) or closes + * prematurely before all mutation groups have been acknowledged, the client automatically + * retries and resumes the request for only the remaining unacknowledged mutation groups. + * The `indexes` field in each emitted {@link protos.google.spanner.v1.BatchWriteResponse|BatchWriteResponse} + * always corresponds to the original positions in the caller's `mutationGroups` array, + * even across retries. + * * @method Spanner#batchWriteAtLeastOnce * * @param {MutationGroup[]} [mutationGroups] The group of mutations to be applied. @@ -3649,45 +3657,143 @@ class Database extends common.GrpcServiceObject { transactionTag: options?.requestOptions?.transactionTag, }, span => { + let dataStream: Transform | null = null; + let spanEnded = false; + const endSpan = (err?: Error | null) => { + if (spanEnded) { + return; + } + spanEnded = true; + if (err) { + setSpanError(span, err); + } + span.end(); + }; + + const originalProxyDestroy = proxyStream._destroy; + proxyStream._destroy = function ( + destroyErr: Error | null, + callback: (error?: Error | null) => void, + ) { + endSpan(destroyErr); + if (dataStream && !dataStream.destroyed) { + dataStream.destroy(destroyErr || undefined); + } + if (typeof originalProxyDestroy === 'function') { + originalProxyDestroy.call(this, destroyErr, callback); + } else { + callback(destroyErr); + } + }; + + proxyStream.once('close', () => { + endSpan(); + if (dataStream && !dataStream.destroyed) { + dataStream.destroy(); + } + }); + + if (!mutationGroups || mutationGroups.length === 0) { + setImmediate(() => { + endSpan(); + proxyStream.end(); + }); + return proxyStream as NodeJS.ReadableStream; + } + this.sessionFactory_.getSession((err, session) => { + if (proxyStream.destroyed) { + endSpan(err); + return; + } + if (err) { + endSpan(err); proxyStream.destroy(err); - setSpanError(span, err); - span.end(); return; } span.addEvent('Using Session', {'session.id': session?.id}); - const gaxOpts = extend(true, {}, options?.gaxOptions); - const reqOpts: spannerClient.spanner.v1.IBatchWriteRequest = { - session: session!.formattedName_!, - mutationGroups: mutationGroups.map(mg => mg.proto()), - requestOptions: options?.requestOptions, - excludeTxnFromChangeStreams: options?.excludeTxnFromChangeStreams, - }; - const headers = this._metadataWithRequestId( - this._nextNthRequest(), - 1, - this.commonHeaders_, - ); - if (this._getSpanner().routeToLeaderEnabled) { - addLeaderAwareRoutingHeader(headers); - } - const dataStream = this.requestStream({ - client: 'SpannerClient', - method: 'batchWrite', - reqOpts, - gaxOpts, - headers, + const gaxOpts = extend(true, {}, options?.gaxOptions, { + retryRequestOptions: { + noResponseRetries: 0, + retries: 0, + }, }); + delete gaxOpts.retry; + const hasExplicitTimeout = + options?.gaxOptions?.timeout !== undefined || + options?.gaxOptions?.retry?.backoffSettings?.totalTimeoutMillis !== + undefined; + const mutationGroupProtos = (mutationGroups ?? []).map(mg => + mg.proto(), + ); + const nthRequest = this._nextNthRequest(); + let attempt = 0; + + const spanner = this._getSpanner(); + let metricsTracer: RequestConfig['metricsTracer']; + + dataStream = batchWriteStream( + mutationGroupProtos, + (currentMutationGroups, remainingTimeoutMs) => { + attempt++; + if (attempt > 1) { + span.addEvent('Resuming BatchWrite stream', { + attempt, + remaining_mutation_groups: currentMutationGroups.length, + }); + } + const reqOpts: spannerClient.spanner.v1.IBatchWriteRequest = { + session: session!.formattedName_!, + mutationGroups: currentMutationGroups, + requestOptions: options?.requestOptions, + excludeTxnFromChangeStreams: + options?.excludeTxnFromChangeStreams, + }; + const headers = this._metadataWithRequestId( + nthRequest, + attempt, + this.commonHeaders_, + ); + if (spanner?.routeToLeaderEnabled) { + addLeaderAwareRoutingHeader(headers); + } + const attemptGaxOpts = + (hasExplicitTimeout || attempt > 1) && + Number.isFinite(remainingTimeoutMs) + ? Object.assign({}, gaxOpts, {timeout: remainingTimeoutMs}) + : gaxOpts; + const requestConfig: RequestConfig = { + client: 'SpannerClient', + method: 'batchWrite', + reqOpts, + gaxOpts: attemptGaxOpts, + headers, + metricsTracer, + deferOperationCompletion: true, + }; + const stream = this.requestStream(requestConfig); + if (metricsTracer === undefined) { + metricsTracer = requestConfig.metricsTracer ?? null; + } + return stream; + }, + { + gaxOptions: options?.gaxOptions, + onTerminate: finalStatus => { + metricsTracer?.recordOperationCompletion(finalStatus); + }, + }, + ); + dataStream .once('error', err => { - setSpanError(span, err); - span.end(); + endSpan(err); proxyStream.destroy(err); }) .once('end', () => { - span.end(); + endSpan(); }) .pipe(proxyStream); }); diff --git a/handwritten/spanner/src/index.ts b/handwritten/spanner/src/index.ts index cf33b3c5af73..6f3aba7b8d1e 100644 --- a/handwritten/spanner/src/index.ts +++ b/handwritten/spanner/src/index.ts @@ -203,7 +203,8 @@ export interface RequestConfig { reqOpts: any; gaxOpts?: CallOptions; headers: {[k: string]: string}; - metricsTracer?: MetricsTracer; + metricsTracer?: MetricsTracer | null; + deferOperationCompletion?: boolean; } export interface CreateInstanceRequest { config?: string; @@ -2135,9 +2136,19 @@ class Spanner extends GrpcService { */ // eslint-disable-next-line @typescript-eslint/no-explicit-any requestStream(config): any { - const metricsTracer = this._initMetricsTracer(config); + const metricsTracer = + config.metricsTracer !== undefined + ? config.metricsTracer + : this._initMetricsTracer(config); + if (config.metricsTracer === undefined && metricsTracer) { + config.metricsTracer = metricsTracer; + } const stream = streamEvents(through.obj()); - const coordinator = new RequestStreamCoordinator(stream, metricsTracer); + const coordinator = new RequestStreamCoordinator( + stream, + metricsTracer, + config.deferOperationCompletion, + ); coordinator.setup(); stream.once('reading', () => { diff --git a/handwritten/spanner/src/metrics/metrics-tracer.ts b/handwritten/spanner/src/metrics/metrics-tracer.ts index 15cf2f34f8e1..ea382960212a 100644 --- a/handwritten/spanner/src/metrics/metrics-tracer.ts +++ b/handwritten/spanner/src/metrics/metrics-tracer.ts @@ -33,10 +33,12 @@ import {isAFEServerTimingEnabled} from '../common'; class MetricAttemptTracer { private _startTime: number; public status: string; + public completed: boolean; constructor() { this._startTime = performance.now(); this.status = Status[Status.UNKNOWN]; + this.completed = false; } /** @@ -327,8 +329,15 @@ export class MetricsTracer { * @param status The status code of the attempt (default: Status.OK). */ public recordAttemptCompletion(statusCode: Status = Status.OK) { - if (!this.enabled || !this.currentOperation?.currentAttempt) return; + if ( + !this.enabled || + !this.currentOperation?.currentAttempt || + this.currentOperation.currentAttempt.completed + ) { + return; + } const currentAttempt = this.currentOperation.currentAttempt; + currentAttempt.completed = true; currentAttempt.status = Status[statusCode] ?? Status[Status.UNKNOWN]; const attemptAttributes = this._createAttemptOtelAttributes(); const endTime = performance.now(); @@ -358,11 +367,22 @@ export class MetricsTracer { /** * Records the completion of the current operation, including its status, * latency, and attempt count. Also clears the current tracer from the factory. + * @param finalStatus Optional final status code override for the operation. */ - public recordOperationCompletion() { + public recordOperationCompletion(finalStatus?: Status) { if (!this.enabled || !this.currentOperation) return; + if ( + finalStatus !== undefined && + this.currentOperation.currentAttempt && + !this.currentOperation.currentAttempt.completed + ) { + this.recordAttemptCompletion(finalStatus); + } const endTime = performance.now(); - const operationAttributes = this._createOperationOtelAttributes(); + const operationAttributes = + finalStatus !== undefined + ? this._getAttributesForStatus(finalStatus) + : this._createOperationOtelAttributes(); const operationLatencyMilliseconds = this._getMillisecondTimeDifference( this.currentOperation.startTime, endTime, diff --git a/handwritten/spanner/src/request-stream-coordinator.ts b/handwritten/spanner/src/request-stream-coordinator.ts index 4254babee75a..79968469a3ea 100644 --- a/handwritten/spanner/src/request-stream-coordinator.ts +++ b/handwritten/spanner/src/request-stream-coordinator.ts @@ -26,16 +26,23 @@ import {MetricsTracer} from './metrics/metrics-tracer'; export class RequestStreamCoordinator { private readonly stream: Transform; private readonly metricsTracer: MetricsTracer | null; + private readonly deferOperationCompletion: boolean; private callStream: CancellableStream | null = null; private statusReceived = false; + private callStreamErrored = false; private streamFinished = false; private operationCompleted = false; private requestStarted = false; private streamClosed = false; - constructor(stream: Transform, metricsTracer: MetricsTracer | null) { + constructor( + stream: Transform, + metricsTracer: MetricsTracer | null, + deferOperationCompletion = false, + ) { this.stream = stream; this.metricsTracer = metricsTracer; + this.deferOperationCompletion = deferOperationCompletion; } /** @@ -100,14 +107,21 @@ export class RequestStreamCoordinator { // This guard preserves OK status for normally drained streams as well as queries // using the PartialResultSet.last optimization (where the user stream completes early // while remaining trailers drain in the background). - if (this.streamFinished || this.statusReceived) { + if (this.streamFinished || this.statusReceived || this.callStreamErrored) { return; } if (this.callStream) { // In google-gax, callStream is a StreamProxy wrapping the underlying gRPC call. // Calling .destroy() on StreamProxy alone does not cancel the active gRPC call; // .cancel() must be invoked explicitly to abort the RPC on the client. - if (typeof this.callStream.cancel === 'function') { + // Guard against StreamProxy wrapping a PassThrough stream (e.g. when client is closed) + // which does not implement .cancel(). + const innerStream = (this.callStream as {stream?: {cancel?: unknown}}) + .stream; + if ( + typeof this.callStream.cancel === 'function' && + (!innerStream || typeof innerStream.cancel === 'function') + ) { this.callStream.cancel(); } // Clean up proxy and stream resources once cancellation has been requested. @@ -162,6 +176,7 @@ export class RequestStreamCoordinator { }); callStream .on('error', (error: Error) => { + this.callStreamErrored = true; if (!this.stream.destroyed && !this.streamClosed) { this.stream.destroy(error); } @@ -221,7 +236,7 @@ export class RequestStreamCoordinator { * Completes the OpenTelemetry operation metric. Idempotent. */ completeOperation(): void { - if (this.operationCompleted) { + if (this.operationCompleted || this.deferOperationCompletion) { return; } this.operationCompleted = true; diff --git a/handwritten/spanner/test/batch-write-stream.ts b/handwritten/spanner/test/batch-write-stream.ts new file mode 100644 index 000000000000..483c277b9d7e --- /dev/null +++ b/handwritten/spanner/test/batch-write-stream.ts @@ -0,0 +1,534 @@ +/*! + * Copyright 2026 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import {beforeEach, afterEach, describe, it} from 'mocha'; +import * as sinon from 'sinon'; +import * as through from 'through2'; +import {Transform} from 'stream'; +import {grpc} from 'google-gax'; +import {protos} from '@google-cloud/spanner-api'; +import {batchWriteStream, getNextRetryDelayMs} from '../src/batch-write-stream'; +import {getRetryInfo} from '../src/protos'; + +type IMutationGroup = protos.google.spanner.v1.BatchWriteRequest.IMutationGroup; +type IBatchWriteResponse = protos.google.spanner.v1.IBatchWriteResponse; + +function makeServiceError( + code: grpc.status, + message: string, + metadata?: grpc.Metadata, +): grpc.ServiceError { + return Object.assign(new Error(message), { + code, + details: message, + metadata: metadata ?? new grpc.Metadata(), + }); +} + +describe('batchWriteStream', () => { + const sandbox = sinon.createSandbox(); + + const group0: IMutationGroup = {mutations: [{insert: {table: 'T0'}}]}; + const group1: IMutationGroup = {mutations: [{insert: {table: 'T1'}}]}; + const group2: IMutationGroup = {mutations: [{insert: {table: 'T2'}}]}; + const group3: IMutationGroup = {mutations: [{insert: {table: 'T3'}}]}; + const group4: IMutationGroup = {mutations: [{insert: {table: 'T4'}}]}; + + const fastGaxOptions = { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + }, + }, + }; + + beforeEach(() => {}); + + afterEach(() => { + sandbox.restore(); + }); + + it('should immediately end stream with OK and zero requests when mutationGroups is empty', done => { + let makeRequestCalled = false; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [], + () => { + makeRequestCalled = true; + return through.obj(); + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(makeRequestCalled, false); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should stream all responses and complete with OK on a single attempt', done => { + const attemptStream = through.obj(); + const receivedRequests: IMutationGroup[][] = []; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 1); + assert.deepStrictEqual(receivedRequests[0], [group0, group1]); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + + attemptStream.write({indexes: [0], status: {code: grpc.status.OK}}); + attemptStream.write({indexes: [1], status: {code: grpc.status.OK}}); + attemptStream.end(); + }); + + it('should retry only unacknowledged mutation groups on UNAVAILABLE and translate indexes across multiple retries', done => { + const attemptStreams: Transform[] = [ + through.obj(), + through.obj(), + through.obj(), + ]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1, group2, group3, group4], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Attempt 1 acknowledges original indexes 0 and 3, then fails with UNAVAILABLE + current.write({indexes: [0, 3], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else if (attemptIndex === 2) { + // Attempt 2 receives [group1, group2, group4] (retry indexes 0, 1, 2 -> original 1, 2, 4) + // Acknowledges retry index 1 (original index 2), then fails with UNAVAILABLE + current.write({indexes: [1], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else if (attemptIndex === 3) { + // Attempt 3 receives [group1, group4] (retry indexes 0, 1 -> original 1, 4) + // Acknowledges retry indexes 0 and 1 + current.write({indexes: [0, 1], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 3); + assert.deepStrictEqual(receivedRequests[0], [ + group0, + group1, + group2, + group3, + group4, + ]); + assert.deepStrictEqual(receivedRequests[1], [group1, group2, group4]); + assert.deepStrictEqual(receivedRequests[2], [group1, group4]); + + assert.strictEqual(responses.length, 3); + assert.deepStrictEqual(responses[0].indexes, [0, 3]); + assert.deepStrictEqual(responses[1].indexes, [2]); + assert.deepStrictEqual(responses[2].indexes, [1, 4]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should not re-send mutation groups that received a non-OK per-group status in BatchWriteResponse', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + + const stream = batchWriteStream( + [group0, group1, group2], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Group 0 fails with per-group ALREADY_EXISTS, Group 1 succeeds with OK + current.write({ + indexes: [0], + status: { + code: grpc.status.ALREADY_EXISTS, + message: 'Row already exists', + }, + }); + current.write({ + indexes: [1], + status: {code: grpc.status.OK}, + }); + // Stream then breaks with UNAVAILABLE before Group 2 is acknowledged + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Stream broken'), + ); + } else { + // Only Group 2 should be sent in Attempt 2 + current.write({ + indexes: [0], + status: {code: grpc.status.OK}, + }); + current.end(); + } + }); + return current; + }, + {gaxOptions: fastGaxOptions}, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[0], [group0, group1, group2]); + assert.deepStrictEqual(receivedRequests[1], [group2]); + assert.strictEqual(responses.length, 3); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.strictEqual( + responses[0].status?.code, + grpc.status.ALREADY_EXISTS, + ); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.deepStrictEqual(responses[2].indexes, [2]); + done(); + }); + }); + + it('should retry on retryable INTERNAL errors (e.g. RST_STREAM or EOS on DATA frame)', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError( + grpc.status.INTERNAL, + 'Received unexpected EOS on DATA frame from server', + ), + ); + } else { + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + {gaxOptions: fastGaxOptions}, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[1], [group1]); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + done(); + }); + }); + + it('should treat premature stream end (with unacknowledged mutation groups) as retryable UNAVAILABLE', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Only acknowledges group0, then closes stream prematurely without error + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } else { + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + {gaxOptions: fastGaxOptions}, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[1], [group1]); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + done(); + }); + }); + + it('should fail withINTERNAL error and not retry when server returns an out-of-bounds index', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err.code, grpc.status.INTERNAL); + assert.match(err.message, /out of bounds/); + assert.strictEqual(finalStatus, grpc.status.INTERNAL); + done(); + }); + + attemptStream.write({indexes: [2], status: {code: grpc.status.OK}}); + }); + + it('should fail immediately without retrying on non-retryable RPC errors', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const permDeniedErr = makeServiceError( + grpc.status.PERMISSION_DENIED, + 'Caller lacks permission', + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err, permDeniedErr); + assert.strictEqual(finalStatus, grpc.status.PERMISSION_DENIED); + done(); + }); + + attemptStream.emit('error', permDeniedErr); + }); + + it('should not retry retryable INTERNAL errors when retryCodes is explicitly empty', done => { + const attemptStream = through.obj(); + let requestCount = 0; + + const stream = batchWriteStream( + [group0], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: { + retry: { + retryCodes: [], + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + }, + }, + }, + }, + ); + + const eosErr = makeServiceError( + grpc.status.INTERNAL, + 'Received unexpected EOS on DATA frame from server', + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err, eosErr); + done(); + }); + + attemptStream.emit('error', eosErr); + }); + + it('should cancel pending retry timer and active stream when caller destroys stream early', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 50, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 100, + totalTimeoutMillis: 5000, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + attemptStream.write({indexes: [0], status: {code: grpc.status.OK}}); + // Trigger retry backoff timer + attemptStream.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Unavailable'), + ); + + // Caller destroys stream while retry timer is waiting + stream.destroy(); + + setTimeout(() => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(finalStatus, grpc.status.CANCELLED); + done(); + }, 80); + }); + + describe('getNextRetryDelayMs', () => { + it('should extract delay from google.rpc.retryinfo-bin metadata when present', () => { + const RetryInfo = getRetryInfo(); + const encoded = RetryInfo.encode({ + retryDelay: {seconds: 2, nanos: 250000000}, + }).finish(); + const metadata = new grpc.Metadata(); + metadata.set('google.rpc.retryinfo-bin', Buffer.from(encoded)); + + const err = makeServiceError( + grpc.status.UNAVAILABLE, + 'Unavailable', + metadata, + ); + const delayMs = getNextRetryDelayMs(err, 1); + assert.strictEqual(delayMs, 2250); + }); + + it('should fall back to jittered exponential backoff when RetryInfo is absent', () => { + const err = makeServiceError(grpc.status.UNAVAILABLE, 'Unavailable'); + const delayMs = getNextRetryDelayMs(err, 1, { + initialRetryDelayMillis: 100, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 32000, + }); + assert.ok( + delayMs >= 80 && delayMs <= 120, + `Expected delay in [80, 120], got ${delayMs}`, + ); + }); + }); +}); diff --git a/handwritten/spanner/test/database.ts b/handwritten/spanner/test/database.ts index d5b66fae4bce..d257c74fe77a 100644 --- a/handwritten/spanner/test/database.ts +++ b/handwritten/spanner/test/database.ts @@ -745,7 +745,12 @@ describe('Database', () => { }); it('should call `requestStream` with correct arguments', () => { - const expectedGaxOpts = extend(true, {}, options?.gaxOptions); + const expectedGaxOpts = extend(true, {}, options?.gaxOptions, { + retryRequestOptions: { + noResponseRetries: 0, + retries: 0, + }, + }); const expectedReqOpts: google.spanner.v1.IBatchWriteRequest = { session: fakeSession!.formattedName_!, mutationGroups: mutationGroups.map(mg => mg.proto()), @@ -789,7 +794,12 @@ describe('Database', () => { assert.strictEqual(args.method, 'batchWrite'); assert.deepStrictEqual(args.reqOpts, expectedReqOpts); assert.strictEqual(args.reqOpts.excludeTxnFromChangeStreams, undefined); - assert.deepStrictEqual(args.gaxOpts, {}); + assert.deepStrictEqual(args.gaxOpts, { + retryRequestOptions: { + noResponseRetries: 0, + retries: 0, + }, + }); }); it('should not include leader-aware routing header when routeToLeaderEnabled is false', () => { @@ -810,13 +820,33 @@ describe('Database', () => { ); }); - it('should return error when passing an empty list of mutationGroups', done => { - const fakeError = new Error('err'); - database.batchWriteAtLeastOnce([], options).on('error', error => { - assert.strictEqual(error, fakeError); - done(); - }); - fakeDataStream.emit('error', fakeError); + it('should immediately end stream with zero RPCs and no session when passing an empty list of mutationGroups', done => { + database + .batchWriteAtLeastOnce([], options) + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(getSessionStub.callCount, 0); + assert.strictEqual(requestStreamStub.callCount, 0); + done(); + }); + }); + + it('should immediately end stream with zero RPCs and no session when mutationGroups is omitted', done => { + database + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .batchWriteAtLeastOnce(undefined as any, options) + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(getSessionStub.callCount, 0); + assert.strictEqual(requestStreamStub.callCount, 0); + done(); + }); }); it('should return data when passing a valid list of mutationGroups', done => { @@ -878,6 +908,175 @@ describe('Database', () => { fakeDataStream.emit('error', fakeError); done(); }); + + it('should retry only remaining unacknowledged mutationGroups on UNAVAILABLE and increment attempt in request ID header', done => { + const stream1 = through.obj(); + const stream2 = through.obj(); + requestStreamStub.onCall(0).returns(stream1); + requestStreamStub.onCall(1).returns(stream2); + + const retryOptions: BatchWriteOptions = { + ...options, + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + }, + }, + }, + }; + + const receivedResponses: google.spanner.v1.IBatchWriteResponse[] = []; + database + .batchWriteAtLeastOnce(mutationGroups, retryOptions) + .on('data', (resp: google.spanner.v1.IBatchWriteResponse) => { + receivedResponses.push(resp); + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(requestStreamStub.callCount, 2); + + const firstCallArgs = requestStreamStub.getCall(0).args[0]; + const secondCallArgs = requestStreamStub.getCall(1).args[0]; + + assert.deepStrictEqual( + firstCallArgs.reqOpts.mutationGroups, + mutationGroups.map(mg => mg.proto()), + ); + assert.strictEqual(firstCallArgs.gaxOpts.retry, undefined); + assert.ok( + typeof firstCallArgs.gaxOpts.timeout === 'number' && + firstCallArgs.gaxOpts.timeout > 0 && + firstCallArgs.gaxOpts.timeout <= 5000, + ); + assert.deepStrictEqual(firstCallArgs.gaxOpts.retryRequestOptions, { + noResponseRetries: 0, + retries: 0, + }); + assert.strictEqual( + firstCallArgs.headers[X_GOOG_SPANNER_REQUEST_ID_HEADER], + craftRequestId(1, 1, 1, 1), + ); + + // Second attempt only sends mutationGroup2 (unacknowledged) + assert.deepStrictEqual(secondCallArgs.reqOpts.mutationGroups, [ + mutationGroup2.proto(), + ]); + assert.ok( + typeof secondCallArgs.gaxOpts.timeout === 'number' && + secondCallArgs.gaxOpts.timeout > 0 && + secondCallArgs.gaxOpts.timeout <= firstCallArgs.gaxOpts.timeout, + ); + // Same nthRequest (1), attempt incremented to 2 + assert.strictEqual( + secondCallArgs.headers[X_GOOG_SPANNER_REQUEST_ID_HEADER], + craftRequestId(1, 1, 1, 2), + ); + + assert.strictEqual(receivedResponses.length, 2); + assert.deepStrictEqual(receivedResponses[0].indexes, [0]); + // Retry-local index 0 in attempt 2 is translated back to original index 1 + assert.deepStrictEqual(receivedResponses[1].indexes, [1]); + done(); + }); + + stream1.write({indexes: [0], status: {code: grpc.status.OK}}); + stream1.emit( + 'error', + Object.assign(new Error('Unavailable'), { + code: grpc.status.UNAVAILABLE, + }), + ); + + setTimeout(() => { + stream2.write({indexes: [0], status: {code: grpc.status.OK}}); + stream2.end(); + }, 15); + }); + + it('should propagate and reuse metricsTracer across retry attempts in batchWriteAtLeastOnce', done => { + const stream1 = through.obj(); + const stream2 = through.obj(); + const fakeTracer = { + recordOperationCompletion: sandbox.spy(), + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let firstAttemptPassedTracer: any; + requestStreamStub.onCall(0).callsFake(config => { + firstAttemptPassedTracer = config.metricsTracer; + config.metricsTracer = fakeTracer; + return stream1; + }); + requestStreamStub.onCall(1).returns(stream2); + + const retryOptions: BatchWriteOptions = { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + }, + }, + }, + }; + + database + .batchWriteAtLeastOnce(mutationGroups, retryOptions) + .on('data', () => {}) + .on('error', done) + .on('end', () => { + assert.strictEqual(requestStreamStub.callCount, 2); + const secondCallArgs = requestStreamStub.getCall(1).args[0]; + + assert.strictEqual(firstAttemptPassedTracer, undefined); + assert.strictEqual(secondCallArgs.metricsTracer, fakeTracer); + assert.strictEqual( + fakeTracer.recordOperationCompletion.calledOnce, + true, + ); + assert.strictEqual( + fakeTracer.recordOperationCompletion.firstCall.args[0], + grpc.status.OK, + ); + done(); + }); + + stream1.write({indexes: [0], status: {code: grpc.status.OK}}); + stream1.emit( + 'error', + Object.assign(new Error('Unavailable'), { + code: grpc.status.UNAVAILABLE, + }), + ); + + setTimeout(() => { + stream2.write({indexes: [0], status: {code: grpc.status.OK}}); + stream2.end(); + }, 15); + }); + + it('should not call `requestStream` or initialize metrics if stream is destroyed before `getSession` resolves', done => { + let sessionCallback: + ((err: Error | null, session?: FakeSession) => void) | null = null; + getSessionStub.callsFake(callback => { + sessionCallback = callback; + }); + + const stream = database.batchWriteAtLeastOnce(mutationGroups, options); + stream.destroy(); + + setImmediate(() => { + assert.ok(sessionCallback); + sessionCallback!(null, fakeSession); + assert.strictEqual(requestStreamStub.callCount, 0); + done(); + }); + }); }); describe('writeAtLeastOnce', () => { diff --git a/handwritten/spanner/test/metrics/metrics-tracer.ts b/handwritten/spanner/test/metrics/metrics-tracer.ts index 9037e3ecdd0a..429effc320e4 100644 --- a/handwritten/spanner/test/metrics/metrics-tracer.ts +++ b/handwritten/spanner/test/metrics/metrics-tracer.ts @@ -281,6 +281,44 @@ describe('MetricsTracer', () => { ); }); + it('should override operation status when finalStatus is passed to recordOperationCompletion', () => { + tracer.recordOperationStart(); + tracer.recordAttemptStart(); + tracer.recordAttemptCompletion(Status.UNAVAILABLE); + tracer.recordOperationCompletion(Status.DEADLINE_EXCEEDED); + + assert.strictEqual(fakeAttemptCounter.add.calledOnce, true); + assert.strictEqual(fakeOperationCounter.add.calledOnce, true); + const [[, operationCounterAttributes]] = fakeOperationCounter.add.args; + assert.strictEqual( + operationCounterAttributes[Constants.METRIC_LABEL_KEY_STATUS], + 'DEADLINE_EXCEEDED', + ); + }); + + it('should finalize an in-flight uncompleted attempt when finalStatus is passed to recordOperationCompletion', () => { + tracer.recordOperationStart(); + tracer.recordAttemptStart(); + // Stream cancelled before recordAttemptCompletion was called by interceptor + tracer.recordOperationCompletion(Status.CANCELLED); + + assert.strictEqual(fakeAttemptCounter.add.calledOnce, true); + assert.strictEqual(fakeAttemptLatency.record.calledOnce, true); + assert.strictEqual(fakeOperationCounter.add.calledOnce, true); + assert.strictEqual(fakeOperationLatency.record.calledOnce, true); + + const [[, attemptAttrs]] = fakeAttemptCounter.add.args; + const [[, operationAttrs]] = fakeOperationCounter.add.args; + assert.strictEqual( + attemptAttrs[Constants.METRIC_LABEL_KEY_STATUS], + 'CANCELLED', + ); + assert.strictEqual( + operationAttrs[Constants.METRIC_LABEL_KEY_STATUS], + 'CANCELLED', + ); + }); + it('should safely do nothing if recordOperationCompletion is called without active operation', () => { tracer.currentOperation = null; assert.doesNotThrow(() => { diff --git a/handwritten/spanner/test/mockserver/mockspanner.ts b/handwritten/spanner/test/mockserver/mockspanner.ts index 4f6f64378d2f..e5937ffb9bf5 100644 --- a/handwritten/spanner/test/mockserver/mockspanner.ts +++ b/handwritten/spanner/test/mockserver/mockspanner.ts @@ -577,6 +577,7 @@ export class MockSpanner { null, protobuf.BatchCreateSessionsResponse.create({session: sessions}), ); + return; }) .catch(err => { callback(err); @@ -597,6 +598,7 @@ export class MockSpanner { call.request!.session?.multiplexed ?? false, ), ); + return; }) .catch(err => { callback(err); @@ -616,6 +618,7 @@ export class MockSpanner { } else { callback(MockSpanner.createSessionNotFoundError(call.request!.name)); } + return; }) .catch(err => { callback(err); @@ -640,6 +643,7 @@ export class MockSpanner { }), }), ); + return; }) .catch(err => { callback(err); @@ -801,6 +805,7 @@ export class MockSpanner { ); } call.end(); + return; }) .catch(err => { call.sendMetadata(new Metadata()); @@ -992,6 +997,7 @@ export class MockSpanner { status: statementStatus, }), ); + return; }) .catch(err => { callback(err); @@ -1123,6 +1129,7 @@ export class MockSpanner { ); } call.end(); + return; }) .catch(err => { call.sendMetadata(new Metadata()); @@ -1151,6 +1158,7 @@ export class MockSpanner { } else { callback(null, res); } + return; }) .catch(err => { callback(err); @@ -1212,6 +1220,7 @@ export class MockSpanner { MockSpanner.createSessionNotFoundError(call.request!.session), ); } + return; }) .catch(err => { callback(err); @@ -1256,6 +1265,7 @@ export class MockSpanner { partitions: [{partitionToken: Buffer.from('mock-token')}], }); callback(null, response); + return; }) .catch(err => callback(err)); } @@ -1271,6 +1281,7 @@ export class MockSpanner { partitions: [{partitionToken: Buffer.from('mock-token')}], }); callback(null, response); + return; }) .catch(err => callback(err)); } @@ -1284,13 +1295,37 @@ export class MockSpanner { this.pushRequest(call.request, call.metadata); this.simulateExecutionTime(this.batchWrite.name) .then(() => { - const response = protobuf.BatchWriteResponse.create({ - commitTimestamp: now(), - }); - call.write(response); + const groupCount = call.request?.mutationGroups?.length ?? 0; + let metadataSent = false; + const ensureMetadataSent = () => { + if (!metadataSent) { + call.sendMetadata(new Metadata()); + metadataSent = true; + } + }; + for (let index = 0; index < groupCount; index++) { + const streamErr = this.shiftStreamError(this.batchWrite.name, index); + if (streamErr) { + ensureMetadataSent(); + call.emit('error', streamErr); + break; + } + ensureMetadataSent(); + const response = protobuf.BatchWriteResponse.create({ + indexes: [index], + status: Status.create({code: grpc.status.OK}), + commitTimestamp: now(), + }); + call.write(response); + } call.end(); + return; }) - .catch(err => call.destroy(err)); + .catch(err => { + call.sendMetadata(new Metadata()); + call.emit('error', err); + call.end(); + }); } private _updateTransaction( diff --git a/handwritten/spanner/test/request-stream-coordinator.ts b/handwritten/spanner/test/request-stream-coordinator.ts index 69c8de388780..d30412843835 100644 --- a/handwritten/spanner/test/request-stream-coordinator.ts +++ b/handwritten/spanner/test/request-stream-coordinator.ts @@ -177,6 +177,108 @@ describe('RequestStreamCoordinator', () => { ); }); + it('should not call recordOperationCompletion when deferOperationCompletion is true', () => { + const stream = streamEvents(through.obj()); + const tracer = createMockTracer(); + const coordinator = new RequestStreamCoordinator(stream, tracer, true); + coordinator.setup(); + + stream.destroy(); + coordinator.completeOperation(); + + assert.strictEqual( + (tracer.recordOperationCompletion as sinon.SinonSpy).called, + false, + ); + }); + + it('should initialize and assign config.metricsTracer when config.metricsTracer is undefined', () => { + const spanner = new Spanner({projectId: 'test-project'}); + const fakeTracer = createMockTracer(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (spanner as any)._initMetricsTracer = sandbox.stub().returns(fakeTracer); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const config: any = {client: 'SpannerClient', method: 'batchWrite'}; + spanner.requestStream(config); + + assert.strictEqual(config.metricsTracer, fakeTracer); + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((spanner as any)._initMetricsTracer as sinon.SinonStub).calledOnce, + true, + ); + }); + + it('should reuse existing config.metricsTracer and not call _initMetricsTracer when config.metricsTracer is provided', () => { + const spanner = new Spanner({projectId: 'test-project'}); + const existingTracer = createMockTracer(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (spanner as any)._initMetricsTracer = sandbox.stub(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const config: any = { + client: 'SpannerClient', + method: 'batchWrite', + metricsTracer: existingTracer, + }; + spanner.requestStream(config); + + assert.strictEqual(config.metricsTracer, existingTracer); + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((spanner as any)._initMetricsTracer as sinon.SinonStub).called, + false, + ); + }); + + it('should preserve config.metricsTracer when set to null and not call _initMetricsTracer', () => { + const spanner = new Spanner({projectId: 'test-project'}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (spanner as any)._initMetricsTracer = sandbox.stub(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const config: any = { + client: 'SpannerClient', + method: 'batchWrite', + metricsTracer: null, + }; + spanner.requestStream(config); + + assert.strictEqual(config.metricsTracer, null); + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((spanner as any)._initMetricsTracer as sinon.SinonStub).called, + false, + ); + }); + + it('should not throw if callStream is a StreamProxy wrapping a PassThrough without cancel()', () => { + const stream = streamEvents(through.obj()); + const tracer = createMockTracer(); + const coordinator = new RequestStreamCoordinator(stream, tracer); + coordinator.setup(); + + const emitter = new EventEmitter(); + const streamProxyWithoutInnerCancel = Object.assign(emitter, { + stream: through.obj(), // PassThrough/Transform without .cancel() + cancel: function () { + // Simulates google-gax StreamProxy.cancel() calling this.stream.cancel() + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this as any).stream.cancel(); + }, + destroy: sandbox.spy(), + pipe: sandbox.spy((destination: unknown) => destination), + destroyed: false, + }) as unknown as CancellableStream; + + coordinator.attachCallStream(streamProxyWithoutInnerCancel); + + assert.doesNotThrow(() => { + stream.destroy(); + }); + }); + it('should invoke requestFn and attach callStream if stream is active', () => { const stream = streamEvents(through.obj()); const tracer = createMockTracer(); @@ -846,4 +948,132 @@ describe('RequestStreamCoordinator with in-memory mock Spanner server', () => { ); assert.strictEqual(operationCancelled?.value, 1); }); + + it('should record 1 operation count and N attempt counts when batchWriteAtLeastOnce retries on UNAVAILABLE', async () => { + const {MutationGroup} = require('../src/transaction'); + const mg0 = new MutationGroup(); + mg0.upsert('FOO', {Id: '1', Name: 'One'}); + const mg1 = new MutationGroup(); + mg1.upsert('FOO', {Id: '2', Name: 'Two'}); + + const unavailableErr = Object.assign(new Error('Mid-stream unavailable'), { + code: grpc.status.UNAVAILABLE, + streamIndex: 1, + metadata: mock.MockSpanner.createMinimalRetryDelayMetadata(), + }) as mock.MockError; + + spannerMock.setExecutionTime( + spannerMock.batchWrite, + mock.SimulatedExecutionTime.ofError(unavailableErr), + ); + + const stream = database.batchWriteAtLeastOnce([mg0, mg1]); + const responses: unknown[] = []; + + await new Promise((resolve, reject) => { + stream.on('data', row => responses.push(row)); + stream.on('end', () => resolve()); + stream.on('error', reject); + }); + + assert.strictEqual(responses.length, 2); + + const resourceMetrics = await waitForMetrics(reader, resMetrics => { + const opPoints = getMetricDataPoints( + resMetrics, + METRIC_NAME_OPERATION_COUNT, + ); + return opPoints.some( + dp => + dp.attributes.method === 'batchWrite' && + dp.attributes.status === 'OK', + ); + }); + + const attemptPoints = getMetricDataPoints( + resourceMetrics, + METRIC_NAME_ATTEMPT_COUNT, + ); + const operationPoints = getMetricDataPoints( + resourceMetrics, + METRIC_NAME_OPERATION_COUNT, + ); + + const attemptUnavailable = findDataPoint( + attemptPoints, + 'batchWrite', + 'UNAVAILABLE', + ); + const attemptOk = findDataPoint(attemptPoints, 'batchWrite', 'OK'); + const operationOk = findDataPoint(operationPoints, 'batchWrite', 'OK'); + const operationUnavailable = findDataPoint( + operationPoints, + 'batchWrite', + 'UNAVAILABLE', + ); + + assert.strictEqual(attemptUnavailable?.value, 1); + assert.strictEqual(attemptOk?.value, 1); + assert.strictEqual(operationOk?.value, 1); + assert.strictEqual(operationUnavailable, undefined); + }); + + it('should record CANCELLED for both attempt and operation when batchWriteAtLeastOnce stream is cancelled early', async () => { + const {MutationGroup} = require('../src/transaction'); + const mg0 = new MutationGroup(); + mg0.upsert('FOO', {Id: '1', Name: 'One'}); + const mg1 = new MutationGroup(); + mg1.upsert('FOO', {Id: '2', Name: 'Two'}); + + const stream = database.batchWriteAtLeastOnce([mg0, mg1]); + + // Read first response and break out of loop (destroying the stream early) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _ of stream) { + break; + } + + const resourceMetrics = await waitForMetrics(reader, resMetrics => { + const opPoints = getMetricDataPoints( + resMetrics, + METRIC_NAME_OPERATION_COUNT, + ); + return opPoints.some( + dp => + dp.attributes.method === 'batchWrite' && + dp.attributes.status === 'CANCELLED', + ); + }); + + const attemptPoints = getMetricDataPoints( + resourceMetrics, + METRIC_NAME_ATTEMPT_COUNT, + ); + const operationPoints = getMetricDataPoints( + resourceMetrics, + METRIC_NAME_OPERATION_COUNT, + ); + + const attemptCancelled = findDataPoint( + attemptPoints, + 'batchWrite', + 'CANCELLED', + ); + const operationCancelled = findDataPoint( + operationPoints, + 'batchWrite', + 'CANCELLED', + ); + + assert.ok( + attemptCancelled, + 'Expected batchWrite attempt with status CANCELLED on early stream destruction', + ); + assert.strictEqual(attemptCancelled?.value, 1); + assert.ok( + operationCancelled, + 'Expected batchWrite operation with status CANCELLED on early stream destruction', + ); + assert.strictEqual(operationCancelled?.value, 1); + }); }); diff --git a/handwritten/spanner/test/spanner.ts b/handwritten/spanner/test/spanner.ts index 8863a78b3eee..4dedee9246d1 100644 --- a/handwritten/spanner/test/spanner.ts +++ b/handwritten/spanner/test/spanner.ts @@ -2278,11 +2278,115 @@ describe('Spanner with mock server', () => { .on('data', response => { // ensure that response is coming assert.notEqual(response.commitTimestamp, null); + assert.deepStrictEqual(response.indexes, [0]); // multiplexed session will get created by default assert.notEqual(multiplexedSession._multiplexedSession, null); }) .on('end', done); }); + + it('should retry mid-stream UNAVAILABLE and resume with only unacknowledged mutation groups', done => { + const mg0 = new MutationGroup(); + mg0.upsert('FOO', {Id: '1', Name: 'One'}); + const mg1 = new MutationGroup(); + mg1.upsert('FOO', {Id: '2', Name: 'Two'}); + const mg2 = new MutationGroup(); + mg2.upsert('FOO', {Id: '3', Name: 'Three'}); + + const unavailableErr = { + message: 'Stream broken mid-flight', + code: grpc.status.UNAVAILABLE, + streamIndex: 1, + metadata: mock.MockSpanner.createMinimalRetryDelayMetadata(), + } as mock.MockError; + + spannerMock.setExecutionTime( + spannerMock.batchWrite, + mock.SimulatedExecutionTime.ofError(unavailableErr), + ); + + const database = newTestDatabase({min: 1, max: 1}); + const responses: protos.google.spanner.v1.IBatchWriteResponse[] = []; + + database + .batchWriteAtLeastOnce([mg0, mg1, mg2], { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 10, + maxRetries: 3, + }, + }, + }, + }) + .on('error', done) + .on( + 'data', + (response: protos.google.spanner.v1.IBatchWriteResponse) => { + responses.push(response); + }, + ) + .on('end', () => { + assert.strictEqual(responses.length, 3); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.deepStrictEqual(responses[2].indexes, [2]); + + const batchWriteRequests = spannerMock + .getRequests() + .filter( + req => + (req as protos.google.spanner.v1.BatchWriteRequest) + .mutationGroups !== undefined, + ) as protos.google.spanner.v1.BatchWriteRequest[]; + + assert.strictEqual(batchWriteRequests.length, 2); + assert.strictEqual(batchWriteRequests[0].mutationGroups.length, 3); + assert.strictEqual(batchWriteRequests[1].mutationGroups.length, 2); + done(); + }); + }); + + it('should not retry immediate non-retryable error in google-gax retry-request', done => { + const mg0 = new MutationGroup(); + mg0.upsert('FOO', {Id: '1', Name: 'One'}); + + const invalidArgErr = { + message: 'Invalid mutation group', + code: grpc.status.INVALID_ARGUMENT, + } as mock.MockError; + + spannerMock.setExecutionTime( + spannerMock.batchWrite, + mock.SimulatedExecutionTime.ofErrors([ + invalidArgErr, + invalidArgErr, + invalidArgErr, + ]), + ); + + const database = newTestDatabase({min: 1, max: 1}); + + database + .batchWriteAtLeastOnce([mg0]) + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', (err: ServiceError) => { + assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT); + const batchWriteRequests = spannerMock + .getRequests() + .filter( + req => + (req as protos.google.spanner.v1.BatchWriteRequest) + .mutationGroups !== undefined, + ) as protos.google.spanner.v1.BatchWriteRequest[]; + assert.strictEqual(batchWriteRequests.length, 1); + done(); + }); + }); }); }); From c757c9f46c57d9c8b0a8c4e996da65d9858b035e Mon Sep 17 00:00:00 2001 From: Alka Trivedi Date: Tue, 3 Nov 2026 07:07:30 +0000 Subject: [PATCH 2/3] addres review comments --- .../spanner/observability-test/database.ts | 5 +- handwritten/spanner/src/batch-write-stream.ts | 573 +++++++++--------- handwritten/spanner/src/common.ts | 62 ++ handwritten/spanner/src/database.ts | 307 ++++++---- handwritten/spanner/src/index.ts | 19 +- handwritten/spanner/src/protos.ts | 5 + .../spanner/src/request-stream-coordinator.ts | 8 +- handwritten/spanner/src/transaction-runner.ts | 6 +- handwritten/spanner/src/transaction.ts | 2 +- .../spanner/test/batch-write-stream.ts | 485 ++++++++++++++- handwritten/spanner/test/database.ts | 26 +- .../spanner/test/mockserver/mockspanner.ts | 3 +- .../test/request-stream-coordinator.ts | 11 +- 13 files changed, 1063 insertions(+), 449 deletions(-) diff --git a/handwritten/spanner/observability-test/database.ts b/handwritten/spanner/observability-test/database.ts index 83c9b008f31d..c6253c7df095 100644 --- a/handwritten/spanner/observability-test/database.ts +++ b/handwritten/spanner/observability-test/database.ts @@ -204,6 +204,7 @@ describe('Database', () => { const SPANNER = { routeToLeaderEnabled: true, + _initMetricsTracer: util.noop, } as {} as Spanner; const INSTANCE = { @@ -1282,8 +1283,8 @@ describe('Database', () => { done(); }); - fakeDataStream.emit('data', 'response'); - fakeDataStream.end('end'); + fakeDataStream.emit('data', {indexes: [0, 1]}); + fakeDataStream.end(); }); it('ends span and does not start RPC when stream is destroyed before getSession resolves', done => { diff --git a/handwritten/spanner/src/batch-write-stream.ts b/handwritten/spanner/src/batch-write-stream.ts index c20144828840..6eba983168c8 100644 --- a/handwritten/spanner/src/batch-write-stream.ts +++ b/handwritten/spanner/src/batch-write-stream.ts @@ -19,76 +19,277 @@ import {Readable, Transform} from 'stream'; import * as through from 'through2'; import {protos} from '@google-cloud/spanner-api'; import {DeadlineError, isRetryableInternalError} from './transaction-runner'; -import {getRetryInfo} from './protos'; - -const RETRY_INFO_BIN = 'google.rpc.retryinfo-bin'; -const RetryInfo = getRetryInfo(); +import {getNextRetryDelayMs} from './common'; type IMutationGroup = protos.google.spanner.v1.BatchWriteRequest.IMutationGroup; type IBatchWriteResponse = protos.google.spanner.v1.IBatchWriteResponse; +type MakeBatchWriteRequest = ( + currentMutationGroups: IMutationGroup[], + remainingTimeoutMs: number, +) => Readable; + export interface BatchWriteStreamOptions { gaxOptions?: CallOptions; onTerminate?: (finalStatus: grpc.status) => void; } -// Default BatchWrite retry/backoff parameters from spanner_client_config.json -const DEFAULT_INITIAL_RETRY_DELAY_MS = 100; -const DEFAULT_MAX_RETRY_DELAY_MS = 60000; -const DEFAULT_RETRY_DELAY_MULTIPLIER = 1.3; +interface BatchWriteState { + makeRequest: MakeBatchWriteRequest; + options: BatchWriteStreamOptions; + userStream: Transform; + pending: Array; + remaining: number; + activeRequestStream: Readable | null; + retryTimer: NodeJS.Timeout | null; + retryCount: number; + startTime: number; + terminated: boolean; +} + const DEFAULT_TOTAL_TIMEOUT_MS = 3600000; // 1 hour /** - * Extracts server-supplied retry delay (in ms) from `google.rpc.retryinfo-bin` - * trailing metadata if present, otherwise computes jittered exponential backoff. + * Checks whether an error is eligible for retry. */ -export function getNextRetryDelayMs( +function isRetryableError( err: grpc.ServiceError, - retryAttempt: number, - backoffSettings?: Partial< - NonNullable['backoffSettings']> - >, -): number { - const retryInfoValues = err?.metadata && err.metadata.get(RETRY_INFO_BIN); - if (retryInfoValues && retryInfoValues.length > 0) { - try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const {retryDelay} = (RetryInfo as any).decode(retryInfoValues[0]); - if (retryDelay) { - let {seconds} = retryDelay; - if ( - seconds && - typeof seconds !== 'number' && - typeof seconds.toNumber === 'function' - ) { - seconds = seconds.toNumber(); - } - const secondsInMs = Math.floor(Number(seconds || 0)) * 1000; - const nanosInMs = Math.floor(Number(retryDelay.nanos || 0)) / 1e6; - const delayMs = secondsInMs + nanosInMs; - if (delayMs >= 0) { - return delayMs; - } + retryCodes?: grpc.status[] | null, +): boolean { + if (!err || typeof err.code !== 'number') { + return false; + } + const retryableCodes = retryCodes ?? [grpc.status.UNAVAILABLE]; + const allowsNetworkRetries = retryableCodes.includes(grpc.status.UNAVAILABLE); + return ( + retryableCodes.includes(err.code) || + (allowsNetworkRetries && isRetryableInternalError(err)) + ); +} + +/** + * Validates response indexes against current request bounds, translates + * retry-local indexes back to original caller positions in-place, and marks + * acknowledged mutation groups as resolved. + */ +function processResponseIndexes( + response: IBatchWriteResponse, + streamIndexes: number[], + pending: Array, +): {error?: grpc.ServiceError; acknowledgedCount: number} { + const respIndexes = response.indexes; + if (!Array.isArray(respIndexes)) { + return {acknowledgedCount: 0}; + } + + // Validate all response indexes before mutating state. + for (let i = 0; i < respIndexes.length; i++) { + const idx = respIndexes[i]; + if ( + typeof idx !== 'number' || + !Number.isInteger(idx) || + idx < 0 || + idx >= streamIndexes.length + ) { + const msg = `BatchWrite index ${idx} out of bounds for current request size ${streamIndexes.length}`; + return { + error: Object.assign(new Error(msg), { + code: grpc.status.INTERNAL, + details: msg, + metadata: new grpc.Metadata(), + } as grpc.ServiceError), + acknowledgedCount: 0, + }; + } + } + + // Translate retry-local indexes back to original caller indexes in-place + // and mark each acknowledged group as resolved (nulling reference for GC). + let acknowledgedCount = 0; + for (let i = 0; i < respIndexes.length; i++) { + const retryIdx = respIndexes[i]; + const origIdx = streamIndexes[retryIdx]; + respIndexes[i] = origIdx; + if (pending[origIdx] !== null) { + pending[origIdx] = null; + acknowledgedCount++; + } + } + + return {acknowledgedCount}; +} + +function finalize(state: BatchWriteState, status: grpc.status): void { + if (state.terminated) { + return; + } + state.terminated = true; + if (state.retryTimer) { + clearTimeout(state.retryTimer); + state.retryTimer = null; + } + state.options.onTerminate?.(status); +} + +function cleanupActiveStream(state: BatchWriteState): void { + if (!state.activeRequestStream) { + return; + } + const streamToClean = state.activeRequestStream; + state.activeRequestStream = null; + streamToClean.removeAllListeners('data'); + streamToClean.removeAllListeners('end'); + streamToClean.removeAllListeners('error'); + streamToClean.on('error', () => {}); + if (!streamToClean.destroyed) { + streamToClean.destroy(); + } +} + +function handleRetryOrFail( + state: BatchWriteState, + err: grpc.ServiceError, +): void { + cleanupActiveStream(state); + if (state.terminated || state.userStream.destroyed) { + return; + } + + const gaxOptions = state.options.gaxOptions; + const backoffSettings = gaxOptions?.retry?.backoffSettings; + const timeout = + gaxOptions?.timeout ?? + backoffSettings?.totalTimeoutMillis ?? + DEFAULT_TOTAL_TIMEOUT_MS; + const maxRetries = backoffSettings?.maxRetries ?? Infinity; + const retryable = isRetryableError(err, gaxOptions?.retry?.retryCodes); + const elapsed = Date.now() - state.startTime; + + if ( + state.remaining > 0 && + state.retryCount < maxRetries && + elapsed < timeout && + retryable + ) { + state.retryCount++; + const delayMs = getNextRetryDelayMs(err, state.retryCount, backoffSettings); + if (elapsed + delayMs >= timeout) { + const deadlineErr = new DeadlineError(err); + finalize(state, grpc.status.DEADLINE_EXCEEDED); + state.userStream.destroy(deadlineErr); + return; + } + state.retryTimer = setTimeout(() => { + state.retryTimer = null; + if (!state.terminated && !state.userStream.destroyed) { + startAttempt(state); } - } catch { - // Fall back to exponential backoff if RetryInfo decoding fails. + }, delayMs); + return; + } + + const finalError = + elapsed >= timeout && retryable ? new DeadlineError(err) : err; + finalize(state, finalError.code ?? grpc.status.UNKNOWN); + state.userStream.destroy(finalError); +} + +function startAttempt(state: BatchWriteState): void { + if (state.terminated || state.userStream.destroyed) { + return; + } + + if (state.remaining === 0) { + finalize(state, grpc.status.OK); + state.userStream.end(); + return; + } + + // Filter unacknowledged mutation groups and record index mapping + const currentGroups: IMutationGroup[] = []; + const streamIndexes: number[] = []; + for (let origIdx = 0; origIdx < state.pending.length; origIdx++) { + const group = state.pending[origIdx]; + if (group !== null) { + currentGroups.push(group); + streamIndexes.push(origIdx); } } - const initialDelay = - backoffSettings?.initialRetryDelayMillis ?? DEFAULT_INITIAL_RETRY_DELAY_MS; - const maxDelay = - backoffSettings?.maxRetryDelayMillis ?? DEFAULT_MAX_RETRY_DELAY_MS; - const multiplier = - backoffSettings?.retryDelayMultiplier ?? DEFAULT_RETRY_DELAY_MULTIPLIER; + const gaxOptions = state.options.gaxOptions; + const timeout = + gaxOptions?.timeout ?? + gaxOptions?.retry?.backoffSettings?.totalTimeoutMillis ?? + DEFAULT_TOTAL_TIMEOUT_MS; + const elapsed = Date.now() - state.startTime; + const remainingTimeoutMs = Number.isFinite(timeout) + ? Math.max(0, timeout - elapsed) + : timeout; + + const currentStream = state.makeRequest(currentGroups, remainingTimeoutMs); + state.activeRequestStream = currentStream; + + currentStream.on('data', (response: IBatchWriteResponse) => { + if (state.terminated || state.userStream.destroyed) { + return; + } + + const {error, acknowledgedCount} = processResponseIndexes( + response, + streamIndexes, + state.pending, + ); + + if (error) { + cleanupActiveStream(state); + finalize(state, grpc.status.INTERNAL); + state.userStream.destroy(error); + return; + } + + state.remaining -= acknowledgedCount; - const unclampedDelay = - initialDelay * Math.pow(multiplier, Math.max(0, retryAttempt - 1)); - const cappedDelay = Math.min(maxDelay, unclampedDelay); + // Forward response and handle downstream backpressure + if (!state.userStream.write(response)) { + currentStream.pause(); + } + }); + + currentStream.once('error', (err: grpc.ServiceError) => { + if (state.terminated || state.userStream.destroyed) { + return; + } + if (state.pending.length > 0 && state.remaining === 0) { + cleanupActiveStream(state); + finalize(state, grpc.status.OK); + state.userStream.end(); + return; + } + handleRetryOrFail(state, err); + }); - // Jitter in [0.8 * cappedDelay, 1.2 * cappedDelay] - const jitterFactor = 0.8 + Math.random() * 0.4; - return Math.floor(cappedDelay * jitterFactor); + currentStream.once('end', () => { + cleanupActiveStream(state); + if (state.terminated || state.userStream.destroyed) { + return; + } + + // Premature stream end (EOF while remaining > 0) is treated as a + // retryable UNAVAILABLE error so remaining groups are retried. + if (state.remaining > 0) { + const msg = `BatchWrite stream ended prematurely with ${state.remaining} unacknowledged mutation group(s)`; + const prematureEofErr = Object.assign(new Error(msg), { + code: grpc.status.UNAVAILABLE, + details: msg, + metadata: new grpc.Metadata(), + } as grpc.ServiceError); + handleRetryOrFail(state, prematureEofErr); + return; + } + + finalize(state, grpc.status.OK); + state.userStream.end(); + }); } /** @@ -98,40 +299,14 @@ export function getNextRetryDelayMs( */ export function batchWriteStream( mutationGroups: IMutationGroup[], - makeRequest: ( - currentMutationGroups: IMutationGroup[], - remainingTimeoutMs: number, - ) => Readable, + makeRequest: MakeBatchWriteRequest, options: BatchWriteStreamOptions = {}, ): Transform { - const gaxOptions = options.gaxOptions; - const backoffSettings = gaxOptions?.retry?.backoffSettings; - const retryableCodes = gaxOptions?.retry?.retryCodes ?? [ - grpc.status.UNAVAILABLE, - ]; - const timeout = - gaxOptions?.timeout ?? - backoffSettings?.totalTimeoutMillis ?? - DEFAULT_TOTAL_TIMEOUT_MS; - const maxRetries = backoffSettings?.maxRetries ?? Infinity; - // Track pending mutation groups by their original caller index [0..N-1]. // Resolved entries are set to `null` so large mutation payloads can be GC'd. - const totalGroups = mutationGroups ? mutationGroups.length : 0; const pending: Array = mutationGroups ? [...mutationGroups] : []; - let remaining = totalGroups; - let sawNonObjectChunk = false; - - // Maps current RPC request index -> original caller index. - let streamIndexes: number[] = []; - - let activeRequestStream: Readable | null = null; - let retryTimer: NodeJS.Timeout | null = null; - let retryCount = 0; - const startTime = Date.now(); - let terminated = false; const userStream: Transform = through.obj( (chunk: IBatchWriteResponse, _enc, next) => { @@ -139,252 +314,58 @@ export function batchWriteStream( }, ); - const finalize = (status: grpc.status) => { - if (terminated) { - return; - } - terminated = true; - if (retryTimer) { - clearTimeout(retryTimer); - retryTimer = null; - } - options.onTerminate?.(status); - }; - - const cleanupActiveRequestStream = () => { - if (!activeRequestStream) { - return; - } - const streamToClean = activeRequestStream; - activeRequestStream = null; - streamToClean.removeAllListeners('data'); - streamToClean.removeAllListeners('end'); - streamToClean.removeAllListeners('error'); - streamToClean.on('error', () => {}); - if (!streamToClean.destroyed) { - streamToClean.destroy(); - } + const state: BatchWriteState = { + makeRequest, + options, + userStream, + pending, + remaining: pending.length, + activeRequestStream: null, + retryTimer: null, + retryCount: 0, + startTime: Date.now(), + terminated: false, }; - // Intercept stream destruction and close so early cancellation by the caller - // immediately clears any pending retry timer and aborts the active RPC stream. + // Hook caller cancellation and cleanup const originalDestroy = userStream._destroy; - userStream._destroy = function ( + userStream._destroy = ( error: Error | null, callback: (error?: Error | null) => void, - ) { - if (!terminated) { - cleanupActiveRequestStream(); + ) => { + if (!state.terminated) { + cleanupActiveStream(state); finalize( + state, (error as grpc.ServiceError | null)?.code ?? grpc.status.CANCELLED, ); } if (typeof originalDestroy === 'function') { - originalDestroy.call(this, error, callback); + originalDestroy.call(userStream, error, callback); } else { callback(error); } }; userStream.once('close', () => { - if (!terminated) { - cleanupActiveRequestStream(); - finalize(grpc.status.CANCELLED); + if (!state.terminated) { + cleanupActiveStream(state); + finalize(state, grpc.status.CANCELLED); } }); - const isRetryableError = (err: grpc.ServiceError): boolean => { - if (!err || typeof err.code !== 'number') { - return false; - } - return ( - retryableCodes.includes(err.code) || - (retryableCodes.includes(grpc.status.UNAVAILABLE) && - isRetryableInternalError(err)) - ); - }; - - const handleRetryOrFail = (err: grpc.ServiceError) => { - cleanupActiveRequestStream(); - if (terminated || userStream.destroyed) { - return; - } - - const elapsed = Date.now() - startTime; - if ( - remaining > 0 && - retryCount < maxRetries && - elapsed < timeout && - isRetryableError(err) - ) { - retryCount++; - const delayMs = getNextRetryDelayMs(err, retryCount, backoffSettings); - if (elapsed + delayMs >= timeout) { - const deadlineErr = new DeadlineError(err); - finalize(grpc.status.DEADLINE_EXCEEDED); - userStream.destroy(deadlineErr); - return; - } - retryTimer = setTimeout(() => { - retryTimer = null; - if (!terminated && !userStream.destroyed) { - startAttempt(); - } - }, delayMs); - return; - } - - const finalError = - elapsed >= timeout && isRetryableError(err) - ? new DeadlineError(err) - : err; - finalize(finalError.code ?? grpc.status.UNKNOWN); - userStream.destroy(finalError); - }; - - const startAttempt = () => { - if (terminated || userStream.destroyed) { - return; - } - - if (remaining === 0) { - finalize(grpc.status.OK); - userStream.end(); - return; - } - - // Build the subset of unacknowledged MutationGroups and their index map. - const currentGroups: IMutationGroup[] = []; - streamIndexes = []; - for (let origIdx = 0; origIdx < totalGroups; origIdx++) { - const group = pending[origIdx]; - if (group !== null) { - currentGroups.push(group); - streamIndexes.push(origIdx); - } - } - - const elapsed = Date.now() - startTime; - const remainingTimeoutMs = Number.isFinite(timeout) - ? Math.max(0, timeout - elapsed) - : timeout; - const currentStream = makeRequest(currentGroups, remainingTimeoutMs); - activeRequestStream = currentStream; - - const onData = (response: IBatchWriteResponse) => { - if (terminated || userStream.destroyed) { - return; - } - - if (typeof response !== 'object' || response === null) { - sawNonObjectChunk = true; - if (!userStream.write(response)) { - currentStream.pause(); - } - return; - } - - const respIndexes = response.indexes; - if (Array.isArray(respIndexes)) { - // Validate all response indexes before mutating state. - for (let i = 0; i < respIndexes.length; i++) { - const idx = Number(respIndexes[i]); - if ( - !Number.isInteger(idx) || - idx < 0 || - idx >= streamIndexes.length - ) { - cleanupActiveRequestStream(); - const msg = `BatchWrite index ${idx} out of bounds for current request size ${streamIndexes.length}`; - const internalErr = Object.assign(new Error(msg), { - code: grpc.status.INTERNAL, - details: msg, - metadata: new grpc.Metadata(), - } as grpc.ServiceError); - finalize(grpc.status.INTERNAL); - userStream.destroy(internalErr); - return; - } - } - - // Translate retry-local indexes back to original caller indexes in-place - // and mark each acknowledged group as resolved (nulling reference for GC). - for (let i = 0; i < respIndexes.length; i++) { - const retryIdx = Number(respIndexes[i]); - const origIdx = streamIndexes[retryIdx]; - respIndexes[i] = origIdx; - if (pending[origIdx] !== null) { - pending[origIdx] = null; - remaining--; - } - } - } - - // Forward the response with translated indexes and respect downstream backpressure. - if (!userStream.write(response)) { - currentStream.pause(); - } - }; - - const onError = (err: grpc.ServiceError) => { - currentStream.removeListener('data', onData); - currentStream.removeListener('end', onEnd); - if (terminated || userStream.destroyed) { - return; - } - if (totalGroups > 0 && remaining === 0 && !sawNonObjectChunk) { - cleanupActiveRequestStream(); - finalize(grpc.status.OK); - userStream.end(); - return; - } - handleRetryOrFail(err); - }; - - const onEnd = () => { - currentStream.removeListener('data', onData); - currentStream.removeListener('error', onError); - activeRequestStream = null; - if (terminated || userStream.destroyed) { - return; - } - - // Premature stream end (EOF while remaining > 0) is treated as a - // retryable UNAVAILABLE error so remaining groups are retried. - if (remaining > 0 && !sawNonObjectChunk) { - const msg = `BatchWrite stream ended prematurely with ${remaining} unacknowledged mutation group(s)`; - const prematureEofErr = Object.assign(new Error(msg), { - code: grpc.status.UNAVAILABLE, - details: msg, - metadata: new grpc.Metadata(), - } as grpc.ServiceError); - handleRetryOrFail(prematureEofErr); - return; - } - - finalize(grpc.status.OK); - userStream.end(); - }; - - currentStream.on('data', onData); - currentStream.once('error', onError); - currentStream.once('end', onEnd); - }; - - // Propagate downstream drain to resume the active gRPC stream. userStream.on('drain', () => { - activeRequestStream?.resume(); + state.activeRequestStream?.resume(); }); - if (totalGroups === 0) { - finalize(grpc.status.OK); + if (pending.length === 0) { + finalize(state, grpc.status.OK); setImmediate(() => { userStream.end(); }); return userStream; } - startAttempt(); - + startAttempt(state); return userStream; } diff --git a/handwritten/spanner/src/common.ts b/handwritten/spanner/src/common.ts index 3dc1be2ed870..6135c7f6c638 100644 --- a/handwritten/spanner/src/common.ts +++ b/handwritten/spanner/src/common.ts @@ -16,6 +16,10 @@ import {grpc, CallOptions, Operation as GaxOperation} from 'google-gax'; import {protos} from '@google-cloud/spanner-api'; +import {getRetryInfo, RETRY_INFO_BIN} from './protos'; + +export {RETRY_INFO_BIN}; +const RetryInfo = getRetryInfo(); import instanceAdmin = protos.google; import databaseAdmin = protos.google; @@ -153,3 +157,61 @@ export function getCommonHeaders( return headers; } + +// Default retry/backoff parameters from spanner_client_config.json +const DEFAULT_INITIAL_RETRY_DELAY_MS = 100; +const DEFAULT_MAX_RETRY_DELAY_MS = 60000; +const DEFAULT_RETRY_DELAY_MULTIPLIER = 1.3; + +/** + * Extracts server-supplied retry delay (in ms) from `google.rpc.retryinfo-bin` + * trailing metadata if present, otherwise computes jittered exponential backoff. + */ +export function getNextRetryDelayMs( + err: grpc.ServiceError, + retryAttempt: number, + backoffSettings?: Partial< + NonNullable['backoffSettings']> + >, +): number { + const retryInfoValues = err?.metadata && err.metadata.get(RETRY_INFO_BIN); + if (retryInfoValues && retryInfoValues.length > 0) { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const {retryDelay} = (RetryInfo as any).decode(retryInfoValues[0]); + if (retryDelay) { + let {seconds} = retryDelay; + if ( + seconds && + typeof seconds !== 'number' && + typeof seconds.toNumber === 'function' + ) { + seconds = seconds.toNumber(); + } + const secondsInMs = Math.floor(Number(seconds || 0)) * 1000; + const nanosInMs = Math.floor(Number(retryDelay.nanos || 0)) / 1e6; + const delayMs = secondsInMs + nanosInMs; + if (delayMs >= 0) { + return delayMs; + } + } + } catch { + // Fall back to exponential backoff if RetryInfo decoding fails. + } + } + + const initialDelay = + backoffSettings?.initialRetryDelayMillis ?? DEFAULT_INITIAL_RETRY_DELAY_MS; + const maxDelay = + backoffSettings?.maxRetryDelayMillis ?? DEFAULT_MAX_RETRY_DELAY_MS; + const multiplier = + backoffSettings?.retryDelayMultiplier ?? DEFAULT_RETRY_DELAY_MULTIPLIER; + + const unclampedDelay = + initialDelay * Math.pow(multiplier, Math.max(0, retryAttempt - 1)); + const cappedDelay = Math.min(maxDelay, unclampedDelay); + + // Jitter in [0.8 * cappedDelay, 1.2 * cappedDelay] + const jitterFactor = 0.8 + Math.random() * 0.4; + return Math.floor(cappedDelay * jitterFactor); +} diff --git a/handwritten/spanner/src/database.ts b/handwritten/spanner/src/database.ts index a5b5cf116554..4544e99193de 100644 --- a/handwritten/spanner/src/database.ts +++ b/handwritten/spanner/src/database.ts @@ -193,6 +193,19 @@ type PoolRequestCallback = RequestCallback; type ResultSetStats = spannerClient.spanner.v1.ResultSetStats; type ResultSetMetadata = spannerClient.spanner.v1.ResultSetMetadata; +type IMutationGroup = spannerClient.spanner.v1.BatchWriteRequest.IMutationGroup; + +interface BatchWriteAttemptParams { + session: Session; + currentMutationGroups: IMutationGroup[]; + remainingTimeoutMs: number; + attempt: number; + nthRequest: number; + baseGaxOpts: CallOptions; + hasExplicitTimeout: boolean; + options?: BatchWriteOptions; + metricsTracer?: RequestConfig['metricsTracer']; +} export type GetSessionsOptions = PagedOptionsWithFilter; export type GetDatabaseRolesOptions = PagedOptionsWithFilter; @@ -3658,40 +3671,11 @@ class Database extends common.GrpcServiceObject { }, span => { let dataStream: Transform | null = null; - let spanEnded = false; - const endSpan = (err?: Error | null) => { - if (spanEnded) { - return; - } - spanEnded = true; - if (err) { - setSpanError(span, err); - } - span.end(); - }; - - const originalProxyDestroy = proxyStream._destroy; - proxyStream._destroy = function ( - destroyErr: Error | null, - callback: (error?: Error | null) => void, - ) { - endSpan(destroyErr); - if (dataStream && !dataStream.destroyed) { - dataStream.destroy(destroyErr || undefined); - } - if (typeof originalProxyDestroy === 'function') { - originalProxyDestroy.call(this, destroyErr, callback); - } else { - callback(destroyErr); - } - }; - - proxyStream.once('close', () => { - endSpan(); - if (dataStream && !dataStream.destroyed) { - dataStream.destroy(); - } - }); + const endSpan = this._bindBatchWriteStreamToSpan( + proxyStream, + span, + () => dataStream, + ); if (!mutationGroups || mutationGroups.length === 0) { setImmediate(() => { @@ -3714,83 +3698,17 @@ class Database extends common.GrpcServiceObject { } span.addEvent('Using Session', {'session.id': session?.id}); - const gaxOpts = extend(true, {}, options?.gaxOptions, { - retryRequestOptions: { - noResponseRetries: 0, - retries: 0, - }, - }); - delete gaxOpts.retry; - const hasExplicitTimeout = - options?.gaxOptions?.timeout !== undefined || - options?.gaxOptions?.retry?.backoffSettings?.totalTimeoutMillis !== - undefined; - const mutationGroupProtos = (mutationGroups ?? []).map(mg => - mg.proto(), - ); - const nthRequest = this._nextNthRequest(); - let attempt = 0; - - const spanner = this._getSpanner(); - let metricsTracer: RequestConfig['metricsTracer']; - - dataStream = batchWriteStream( - mutationGroupProtos, - (currentMutationGroups, remainingTimeoutMs) => { - attempt++; - if (attempt > 1) { - span.addEvent('Resuming BatchWrite stream', { - attempt, - remaining_mutation_groups: currentMutationGroups.length, - }); - } - const reqOpts: spannerClient.spanner.v1.IBatchWriteRequest = { - session: session!.formattedName_!, - mutationGroups: currentMutationGroups, - requestOptions: options?.requestOptions, - excludeTxnFromChangeStreams: - options?.excludeTxnFromChangeStreams, - }; - const headers = this._metadataWithRequestId( - nthRequest, - attempt, - this.commonHeaders_, - ); - if (spanner?.routeToLeaderEnabled) { - addLeaderAwareRoutingHeader(headers); - } - const attemptGaxOpts = - (hasExplicitTimeout || attempt > 1) && - Number.isFinite(remainingTimeoutMs) - ? Object.assign({}, gaxOpts, {timeout: remainingTimeoutMs}) - : gaxOpts; - const requestConfig: RequestConfig = { - client: 'SpannerClient', - method: 'batchWrite', - reqOpts, - gaxOpts: attemptGaxOpts, - headers, - metricsTracer, - deferOperationCompletion: true, - }; - const stream = this.requestStream(requestConfig); - if (metricsTracer === undefined) { - metricsTracer = requestConfig.metricsTracer ?? null; - } - return stream; - }, - { - gaxOptions: options?.gaxOptions, - onTerminate: finalStatus => { - metricsTracer?.recordOperationCompletion(finalStatus); - }, - }, + dataStream = this._createBatchWriteStream( + session!, + mutationGroups, + options, + span, ); dataStream - .once('error', err => { - endSpan(err); - proxyStream.destroy(err); + .once('error', streamErr => { + endSpan(streamErr); + proxyStream.destroy(streamErr); }) .once('end', () => { endSpan(); @@ -3803,6 +3721,167 @@ class Database extends common.GrpcServiceObject { ); } + /** + * Attaches destruction and close listeners to the user-facing proxy stream + * to ensure the trace span ends and the active gRPC stream is destroyed on early abort. + */ + private _bindBatchWriteStreamToSpan( + proxyStream: Transform, + span: Span, + getDataStream: () => Transform | null, + ): (err?: Error | null) => void { + let spanEnded = false; + const endSpan = (err?: Error | null) => { + if (spanEnded) { + return; + } + spanEnded = true; + if (err) { + setSpanError(span, err); + } + span.end(); + }; + + const originalProxyDestroy = proxyStream._destroy; + proxyStream._destroy = function ( + destroyErr: Error | null, + callback: (error?: Error | null) => void, + ) { + endSpan(destroyErr); + const dataStream = getDataStream(); + if (dataStream && !dataStream.destroyed) { + dataStream.destroy(destroyErr || undefined); + } + if (typeof originalProxyDestroy === 'function') { + originalProxyDestroy.call(this, destroyErr, callback); + } else { + callback(destroyErr); + } + }; + + proxyStream.once('close', () => { + endSpan(); + const dataStream = getDataStream(); + if (dataStream && !dataStream.destroyed) { + dataStream.destroy(); + } + }); + + return endSpan; + } + + /** + * Issues an individual BatchWrite attempt RPC with appropriate timeout, + * request headers, and metrics tracing. + */ + private _createBatchWriteAttempt({ + session, + currentMutationGroups, + remainingTimeoutMs, + attempt, + nthRequest, + baseGaxOpts, + hasExplicitTimeout, + options, + metricsTracer, + }: BatchWriteAttemptParams): Readable { + const reqOpts: spannerClient.spanner.v1.IBatchWriteRequest = { + session: session.formattedName_!, + mutationGroups: currentMutationGroups, + requestOptions: options?.requestOptions, + excludeTxnFromChangeStreams: options?.excludeTxnFromChangeStreams, + }; + const headers = this._metadataWithRequestId( + nthRequest, + attempt, + this.commonHeaders_, + ); + if (this._getSpanner()?.routeToLeaderEnabled) { + addLeaderAwareRoutingHeader(headers); + } + const attemptGaxOpts = + (hasExplicitTimeout || attempt > 1) && Number.isFinite(remainingTimeoutMs) + ? Object.assign({}, baseGaxOpts, {timeout: remainingTimeoutMs}) + : baseGaxOpts; + const requestConfig: RequestConfig = { + client: 'SpannerClient', + method: 'batchWrite', + reqOpts, + gaxOpts: attemptGaxOpts, + headers, + metricsTracer, + }; + return this.requestStream(requestConfig); + } + + /** + * Creates the resilient BatchWrite stream for an active session, coordinating + * retries across attempts and recording final metrics upon termination. + */ + private _createBatchWriteStream( + session: Session, + mutationGroups: MutationGroup[], + options: BatchWriteOptions | undefined, + span: Span, + ): Transform { + const baseGaxOpts = extend(true, {}, options?.gaxOptions, { + retryRequestOptions: { + noResponseRetries: 0, + retries: 0, + }, + }); + delete baseGaxOpts.retry; + const hasExplicitTimeout = + options?.gaxOptions?.timeout !== undefined || + options?.gaxOptions?.retry?.backoffSettings?.totalTimeoutMillis !== + undefined; + const mutationGroupProtos = (mutationGroups ?? []).map(mg => mg.proto()); + const nthRequest = this._nextNthRequest(); + let attempt = 0; + + const initialHeaders = this._metadataWithRequestId( + nthRequest, + 1, + this.commonHeaders_, + ); + const metricsTracer = this._initMetricsTracer({ + client: 'SpannerClient', + method: 'batchWrite', + reqOpts: {session: session.formattedName_!}, + headers: initialHeaders, + }); + + return batchWriteStream( + mutationGroupProtos, + (currentMutationGroups, remainingTimeoutMs) => { + attempt++; + if (attempt > 1) { + span.addEvent('Resuming BatchWrite stream', { + attempt, + remaining_mutation_groups: currentMutationGroups.length, + }); + } + return this._createBatchWriteAttempt({ + session, + currentMutationGroups, + remainingTimeoutMs, + attempt, + nthRequest, + baseGaxOpts, + hasExplicitTimeout, + options, + metricsTracer, + }); + }, + { + gaxOptions: options?.gaxOptions, + onTerminate: finalStatus => { + metricsTracer?.recordOperationCompletion(finalStatus); + }, + }, + ); + } + /** * Write mutations using a single RPC invocation without replay protection. * @@ -4212,6 +4291,18 @@ class Database extends common.GrpcServiceObject { private _getSpanner(): Spanner { return this.instance.parent as Spanner; } + + private _initMetricsTracer( + config: RequestConfig, + ): RequestConfig['metricsTracer'] { + const spanner = this._getSpanner(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (typeof (spanner as any)?._initMetricsTracer === 'function') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (spanner as any)._initMetricsTracer(config) ?? undefined; + } + return undefined; + } } /*! Developer Documentation diff --git a/handwritten/spanner/src/index.ts b/handwritten/spanner/src/index.ts index 6f3aba7b8d1e..28102ba00d86 100644 --- a/handwritten/spanner/src/index.ts +++ b/handwritten/spanner/src/index.ts @@ -203,8 +203,7 @@ export interface RequestConfig { reqOpts: any; gaxOpts?: CallOptions; headers: {[k: string]: string}; - metricsTracer?: MetricsTracer | null; - deferOperationCompletion?: boolean; + metricsTracer?: MetricsTracer; } export interface CreateInstanceRequest { config?: string; @@ -2136,18 +2135,18 @@ class Spanner extends GrpcService { */ // eslint-disable-next-line @typescript-eslint/no-explicit-any requestStream(config): any { - const metricsTracer = - config.metricsTracer !== undefined - ? config.metricsTracer - : this._initMetricsTracer(config); - if (config.metricsTracer === undefined && metricsTracer) { - config.metricsTracer = metricsTracer; - } + const isExternalTracer = + config !== null && + config !== undefined && + Object.prototype.hasOwnProperty.call(config, 'metricsTracer'); + const metricsTracer = isExternalTracer + ? (config.metricsTracer ?? null) + : this._initMetricsTracer(config); const stream = streamEvents(through.obj()); const coordinator = new RequestStreamCoordinator( stream, metricsTracer, - config.deferOperationCompletion, + isExternalTracer, ); coordinator.setup(); diff --git a/handwritten/spanner/src/protos.ts b/handwritten/spanner/src/protos.ts index 79cfd2f3cc3f..bcca30f96eed 100644 --- a/handwritten/spanner/src/protos.ts +++ b/handwritten/spanner/src/protos.ts @@ -25,6 +25,11 @@ const RetryInfo = Root.fromJSON(statusProtoJson).lookupType( 'google.rpc.RetryInfo', ); +/** + * Binary metadata key used by gRPC to transmit RetryInfo. + */ +export const RETRY_INFO_BIN = 'google.rpc.retryinfo-bin'; + /** * Builds the filesystem path to a particular proto sub-path, for protos * stored in `@google-cloud/spanner-api`. diff --git a/handwritten/spanner/src/request-stream-coordinator.ts b/handwritten/spanner/src/request-stream-coordinator.ts index 79968469a3ea..ca88ac909751 100644 --- a/handwritten/spanner/src/request-stream-coordinator.ts +++ b/handwritten/spanner/src/request-stream-coordinator.ts @@ -26,7 +26,7 @@ import {MetricsTracer} from './metrics/metrics-tracer'; export class RequestStreamCoordinator { private readonly stream: Transform; private readonly metricsTracer: MetricsTracer | null; - private readonly deferOperationCompletion: boolean; + private readonly isExternalTracer: boolean; private callStream: CancellableStream | null = null; private statusReceived = false; private callStreamErrored = false; @@ -38,11 +38,11 @@ export class RequestStreamCoordinator { constructor( stream: Transform, metricsTracer: MetricsTracer | null, - deferOperationCompletion = false, + isExternalTracer = false, ) { this.stream = stream; this.metricsTracer = metricsTracer; - this.deferOperationCompletion = deferOperationCompletion; + this.isExternalTracer = isExternalTracer; } /** @@ -236,7 +236,7 @@ export class RequestStreamCoordinator { * Completes the OpenTelemetry operation metric. Idempotent. */ completeOperation(): void { - if (this.operationCompleted || this.deferOperationCompletion) { + if (this.operationCompleted || this.isExternalTracer) { return; } this.operationCompleted = true; diff --git a/handwritten/spanner/src/transaction-runner.ts b/handwritten/spanner/src/transaction-runner.ts index b1921eedb0f2..c9e721bf9d55 100644 --- a/handwritten/spanner/src/transaction-runner.ts +++ b/handwritten/spanner/src/transaction-runner.ts @@ -27,9 +27,7 @@ import google = protos.google; import IRequestOptions = google.spanner.v1.IRequestOptions; import IsolationLevel = google.spanner.v1.TransactionOptions.IsolationLevel; import ReadLockMode = google.spanner.v1.TransactionOptions.ReadWrite.ReadLockMode; -import {getRetryInfo} from './protos'; - -const RETRY_INFO = 'google.rpc.retryinfo-bin'; +import {getRetryInfo, RETRY_INFO_BIN} from './protos'; const RETRYABLE: grpc.status[] = [grpc.status.ABORTED]; @@ -157,7 +155,7 @@ export abstract class Runner { * @returns {number} Delay in milliseconds. */ getNextDelay(err: grpc.ServiceError): number { - const retryInfo = err.metadata && err.metadata.get(RETRY_INFO); + const retryInfo = err.metadata && err.metadata.get(RETRY_INFO_BIN); if (retryInfo && retryInfo.length) { // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/handwritten/spanner/src/transaction.ts b/handwritten/spanner/src/transaction.ts index 6e546f62bf14..e62e55a321d6 100644 --- a/handwritten/spanner/src/transaction.ts +++ b/handwritten/spanner/src/transaction.ts @@ -67,12 +67,12 @@ import { X_GOOG_SPANNER_REQUEST_ID_HEADER, X_GOOG_SPANNER_REQUEST_ID_SPAN_ATTR, } from './request_id_header'; +import {RETRY_INFO_BIN} from './protos'; export type Rows = Array; type ResultStats = ResultSetStats | IResultSetStats; type ResultMetadata = ResultSetMetadata | IResultSetMetadata; const RETRY_INFO_TYPE = 'type.googleapis.com/google.rpc.retryinfo'; -const RETRY_INFO_BIN = 'google.rpc.retryinfo-bin'; let nextAffinityId = 0; diff --git a/handwritten/spanner/test/batch-write-stream.ts b/handwritten/spanner/test/batch-write-stream.ts index 483c277b9d7e..4a4c005c8777 100644 --- a/handwritten/spanner/test/batch-write-stream.ts +++ b/handwritten/spanner/test/batch-write-stream.ts @@ -21,8 +21,10 @@ import * as through from 'through2'; import {Transform} from 'stream'; import {grpc} from 'google-gax'; import {protos} from '@google-cloud/spanner-api'; -import {batchWriteStream, getNextRetryDelayMs} from '../src/batch-write-stream'; -import {getRetryInfo} from '../src/protos'; +import {batchWriteStream} from '../src/batch-write-stream'; +import {getNextRetryDelayMs} from '../src/common'; +import {getRetryInfo, RETRY_INFO_BIN} from '../src/protos'; +import {DeadlineError} from '../src/transaction-runner'; type IMutationGroup = protos.google.spanner.v1.BatchWriteRequest.IMutationGroup; type IBatchWriteResponse = protos.google.spanner.v1.IBatchWriteResponse; @@ -352,7 +354,107 @@ describe('batchWriteStream', () => { }); }); - it('should fail withINTERNAL error and not retry when server returns an out-of-bounds index', done => { + it('should detach error listener on premature stream end so late errors do not trigger duplicate retries', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Acknowledge group0, end prematurely, then emit late errors on the closed stream + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + setImmediate(() => { + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Late disconnect'), + ); + current.emit( + 'error', + makeServiceError(grpc.status.CANCELLED, 'Trailing cancel'), + ); + }); + } else { + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[1], [group1]); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should finalize with OK and end stream when trailing gRPC error occurs after all mutation groups are acknowledged', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + setImmediate(() => { + attemptStream.write({indexes: [0], status: {code: grpc.status.OK}}); + attemptStream.write({indexes: [1], status: {code: grpc.status.OK}}); + attemptStream.emit( + 'error', + makeServiceError( + grpc.status.UNAVAILABLE, + 'Connection reset by peer', + ), + ); + }); + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should fail with INTERNAL error and not retry when server returns an out-of-bounds index', done => { const attemptStream = through.obj(); let requestCount = 0; let finalStatus: grpc.status | undefined; @@ -382,6 +484,154 @@ describe('batchWriteStream', () => { attemptStream.write({indexes: [2], status: {code: grpc.status.OK}}); }); + it('should fail with INTERNAL error and not retry when server returns an out-of-bounds index on a resumed stream', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1, group2], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Acknowledge group0, then fail with UNAVAILABLE + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else { + // Resumed stream only sent 2 groups ([group1, group2]), so index 2 is out of bounds + current.write({indexes: [2], status: {code: grpc.status.OK}}); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', (err: grpc.ServiceError) => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[0], [group0, group1, group2]); + assert.deepStrictEqual(receivedRequests[1], [group1, group2]); + assert.strictEqual(responses.length, 1); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.strictEqual(err.code, grpc.status.INTERNAL); + assert.match(err.message, /out of bounds/); + assert.strictEqual(finalStatus, grpc.status.INTERNAL); + done(); + }); + }); + + for (const invalidIndex of [-1, 1.5, null as unknown as number]) { + it(`should fail with INTERNAL error and not retry on invalid index (${String(invalidIndex)})`, done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err.code, grpc.status.INTERNAL); + assert.match(err.message, /out of bounds/); + assert.strictEqual(finalStatus, grpc.status.INTERNAL); + done(); + }); + + attemptStream.write({ + indexes: [invalidIndex], + status: {code: grpc.status.OK}, + }); + }); + } + + it('should translate duplicate indexes in-place and decrement remaining only once per unique group', done => { + const attemptStreams: Transform[] = [ + through.obj(), + through.obj(), + through.obj(), + ]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1, group2], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Acknowledge group0, then fail with UNAVAILABLE + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else if (attemptIndex === 2) { + // Resumed stream has [group1, group2] (2 groups). + // Server sends duplicate index [0, 0] for group1 and ends prematurely. + // Must NOT decrement remaining to 0 (group2 is still unacknowledged). + current.write({indexes: [0, 0], status: {code: grpc.status.OK}}); + current.end(); + } else { + // Attempt 3 must resume with only [group2] + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 3); + assert.deepStrictEqual(receivedRequests[0], [group0, group1, group2]); + assert.deepStrictEqual(receivedRequests[1], [group1, group2]); + assert.deepStrictEqual(receivedRequests[2], [group2]); + assert.strictEqual(responses.length, 3); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1, 1]); + assert.deepStrictEqual(responses[2].indexes, [2]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + it('should fail immediately without retrying on non-retryable RPC errors', done => { const attemptStream = through.obj(); let requestCount = 0; @@ -416,6 +666,60 @@ describe('batchWriteStream', () => { attemptStream.emit('error', permDeniedErr); }); + it('should emit non-retryable error and stop retrying after partial retry progress', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const permDeniedErr = makeServiceError( + grpc.status.PERMISSION_DENIED, + 'Caller lacks permission on table T1', + ); + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Attempt 1 acknowledges group0, then fails with UNAVAILABLE + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else { + // Attempt 2 receives only [group1] and fails with non-retryable PERMISSION_DENIED + current.emit('error', permDeniedErr); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', (err: grpc.ServiceError) => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[0], [group0, group1]); + assert.deepStrictEqual(receivedRequests[1], [group1]); + assert.strictEqual(responses.length, 1); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.strictEqual(err, permDeniedErr); + assert.strictEqual(finalStatus, grpc.status.PERMISSION_DENIED); + done(); + }); + }); + it('should not retry retryable INTERNAL errors when retryCodes is explicitly empty', done => { const attemptStream = through.obj(); let requestCount = 0; @@ -500,6 +804,159 @@ describe('batchWriteStream', () => { }, 80); }); + it('should emit DeadlineError and finalize with DEADLINE_EXCEEDED when retry delay exceeds total timeout', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0], + () => { + requestCount++; + setImmediate(() => { + attemptStream.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + }); + return attemptStream; + }, + { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1000, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 2000, + totalTimeoutMillis: 50, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.ok(err instanceof DeadlineError); + assert.strictEqual(err.code, grpc.status.DEADLINE_EXCEEDED); + assert.strictEqual(finalStatus, grpc.status.DEADLINE_EXCEEDED); + assert.strictEqual((err as DeadlineError).errors.length, 1); + assert.strictEqual( + (err as DeadlineError).errors[0].code, + grpc.status.UNAVAILABLE, + ); + done(); + }); + }); + + it('should emit DeadlineError and finalize with DEADLINE_EXCEEDED when total timeout has already elapsed prior to failure', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + let currentTime = 1000; + sandbox.stub(Date, 'now').callsFake(() => currentTime); + + const stream = batchWriteStream( + [group0], + () => { + requestCount++; + setImmediate(() => { + // Advance time past the 500ms timeout + currentTime += 600; + attemptStream.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + }); + return attemptStream; + }, + { + gaxOptions: { + timeout: 500, + retry: { + backoffSettings: { + initialRetryDelayMillis: 10, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 50, + totalTimeoutMillis: 500, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.ok(err instanceof DeadlineError); + assert.strictEqual(err.code, grpc.status.DEADLINE_EXCEEDED); + assert.strictEqual(finalStatus, grpc.status.DEADLINE_EXCEEDED); + assert.strictEqual((err as DeadlineError).errors.length, 1); + assert.strictEqual( + (err as DeadlineError).errors[0].code, + grpc.status.UNAVAILABLE, + ); + done(); + }); + }); + + it('should cease retrying and emit final error when maxRetries is exhausted', done => { + const attemptStreams: Transform[] = [ + through.obj(), + through.obj(), + through.obj(), + ]; + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const unavailableErr = makeServiceError( + grpc.status.UNAVAILABLE, + 'Service currently unavailable', + ); + + const stream = batchWriteStream( + [group0], + () => { + const current = attemptStreams[requestCount++]; + setImmediate(() => { + current.emit('error', unavailableErr); + }); + return current; + }, + { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + maxRetries: 2, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + // 1 initial attempt + 2 retries = 3 attempts total + assert.strictEqual(requestCount, 3); + assert.strictEqual(err, unavailableErr); + assert.strictEqual(err.code, grpc.status.UNAVAILABLE); + assert.strictEqual(finalStatus, grpc.status.UNAVAILABLE); + done(); + }); + }); + describe('getNextRetryDelayMs', () => { it('should extract delay from google.rpc.retryinfo-bin metadata when present', () => { const RetryInfo = getRetryInfo(); @@ -507,7 +964,7 @@ describe('batchWriteStream', () => { retryDelay: {seconds: 2, nanos: 250000000}, }).finish(); const metadata = new grpc.Metadata(); - metadata.set('google.rpc.retryinfo-bin', Buffer.from(encoded)); + metadata.set(RETRY_INFO_BIN, Buffer.from(encoded)); const err = makeServiceError( grpc.status.UNAVAILABLE, @@ -530,5 +987,25 @@ describe('batchWriteStream', () => { `Expected delay in [80, 120], got ${delayMs}`, ); }); + + it('should fall back to jittered exponential backoff when RetryInfo metadata is corrupted', () => { + const metadata = new grpc.Metadata(); + metadata.set(RETRY_INFO_BIN, Buffer.from('invalid-non-proto-bytes')); + + const err = makeServiceError( + grpc.status.UNAVAILABLE, + 'Unavailable', + metadata, + ); + const delayMs = getNextRetryDelayMs(err, 1, { + initialRetryDelayMillis: 100, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 32000, + }); + assert.ok( + delayMs >= 80 && delayMs <= 120, + `Expected delay in [80, 120], got ${delayMs}`, + ); + }); }); }); diff --git a/handwritten/spanner/test/database.ts b/handwritten/spanner/test/database.ts index d257c74fe77a..ebc24945d80c 100644 --- a/handwritten/spanner/test/database.ts +++ b/handwritten/spanner/test/database.ts @@ -276,6 +276,7 @@ describe('Database', () => { const SPANNER = { routeToLeaderEnabled: true, options: {}, + _initMetricsTracer: util.noop, } as {} as Spanner; const INSTANCE = { @@ -850,13 +851,14 @@ describe('Database', () => { }); it('should return data when passing a valid list of mutationGroups', done => { + const fakeResponse = {indexes: [0, 1]}; database .batchWriteAtLeastOnce(mutationGroups, options) .on('data', data => { - assert.strictEqual(data, 'test'); + assert.deepStrictEqual(data, fakeResponse); done(); }); - fakeDataStream.emit('data', 'test'); + fakeDataStream.emit('data', fakeResponse); }); it('should emit correct event based on valid/invalid list of mutationGroups', done => { @@ -893,18 +895,19 @@ describe('Database', () => { SingerId: 3, AlbumTitle: 'Go, Go, Go', }); + const fakeResponse = {indexes: [0]}; database .batchWriteAtLeastOnce( [FakeMutationGroup1, FakeMutationGroup2], options, ) .on('data', data => { - assert.strictEqual(data, 'testData'); + assert.deepStrictEqual(data, fakeResponse); }) .on('error', err => { assert.strictEqual(err, fakeError); }); - fakeDataStream.emit('data', 'testData'); + fakeDataStream.emit('data', fakeResponse); fakeDataStream.emit('error', fakeError); done(); }); @@ -997,19 +1000,17 @@ describe('Database', () => { }, 15); }); - it('should propagate and reuse metricsTracer across retry attempts in batchWriteAtLeastOnce', done => { + it('should initialize and pass metricsTracer across all retry attempts in batchWriteAtLeastOnce', done => { const stream1 = through.obj(); const stream2 = through.obj(); const fakeTracer = { recordOperationCompletion: sandbox.spy(), }; // eslint-disable-next-line @typescript-eslint/no-explicit-any - let firstAttemptPassedTracer: any; - requestStreamStub.onCall(0).callsFake(config => { - firstAttemptPassedTracer = config.metricsTracer; - config.metricsTracer = fakeTracer; - return stream1; - }); + const spanner = (database as any)._getSpanner(); + sandbox.stub(spanner, '_initMetricsTracer').returns(fakeTracer); + + requestStreamStub.onCall(0).returns(stream1); requestStreamStub.onCall(1).returns(stream2); const retryOptions: BatchWriteOptions = { @@ -1031,9 +1032,10 @@ describe('Database', () => { .on('error', done) .on('end', () => { assert.strictEqual(requestStreamStub.callCount, 2); + const firstCallArgs = requestStreamStub.getCall(0).args[0]; const secondCallArgs = requestStreamStub.getCall(1).args[0]; - assert.strictEqual(firstAttemptPassedTracer, undefined); + assert.strictEqual(firstCallArgs.metricsTracer, fakeTracer); assert.strictEqual(secondCallArgs.metricsTracer, fakeTracer); assert.strictEqual( fakeTracer.recordOperationCompletion.calledOnce, diff --git a/handwritten/spanner/test/mockserver/mockspanner.ts b/handwritten/spanner/test/mockserver/mockspanner.ts index e5937ffb9bf5..d245800ded1e 100644 --- a/handwritten/spanner/test/mockserver/mockspanner.ts +++ b/handwritten/spanner/test/mockserver/mockspanner.ts @@ -31,7 +31,7 @@ import Any = google.protobuf.Any; import QueryMode = google.spanner.v1.ExecuteSqlRequest.QueryMode; import NullValue = google.protobuf.NullValue; import {ExecuteSqlRequest, ReadRequest} from '../../src/transaction'; -import {getProtoPath, getRetryInfo} from '../../src/protos'; +import {getProtoPath, getRetryInfo, RETRY_INFO_BIN} from '../../src/protos'; const RetryInfo = getRetryInfo(); @@ -57,7 +57,6 @@ const packageDefinition = protoLoader.loadSync(PROTO_PATH, { }); const protoDescriptor = grpc.loadPackageDefinition(packageDefinition); const spannerProtoDescriptor = protoDescriptor['google']['spanner']['v1']; -const RETRY_INFO_BIN = 'google.rpc.retryinfo-bin'; const RETRY_INFO_TYPE = 'type.googleapis.com/google.rpc.retryinfo'; /** diff --git a/handwritten/spanner/test/request-stream-coordinator.ts b/handwritten/spanner/test/request-stream-coordinator.ts index d30412843835..27d05afb8eda 100644 --- a/handwritten/spanner/test/request-stream-coordinator.ts +++ b/handwritten/spanner/test/request-stream-coordinator.ts @@ -177,7 +177,7 @@ describe('RequestStreamCoordinator', () => { ); }); - it('should not call recordOperationCompletion when deferOperationCompletion is true', () => { + it('should not call recordOperationCompletion when isExternalTracer is true', () => { const stream = streamEvents(through.obj()); const tracer = createMockTracer(); const coordinator = new RequestStreamCoordinator(stream, tracer, true); @@ -192,7 +192,7 @@ describe('RequestStreamCoordinator', () => { ); }); - it('should initialize and assign config.metricsTracer when config.metricsTracer is undefined', () => { + it('should call _initMetricsTracer when config.metricsTracer is undefined', () => { const spanner = new Spanner({projectId: 'test-project'}); const fakeTracer = createMockTracer(); // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -202,7 +202,6 @@ describe('RequestStreamCoordinator', () => { const config: any = {client: 'SpannerClient', method: 'batchWrite'}; spanner.requestStream(config); - assert.strictEqual(config.metricsTracer, fakeTracer); assert.strictEqual( // eslint-disable-next-line @typescript-eslint/no-explicit-any ((spanner as any)._initMetricsTracer as sinon.SinonStub).calledOnce, @@ -232,7 +231,7 @@ describe('RequestStreamCoordinator', () => { ); }); - it('should preserve config.metricsTracer when set to null and not call _initMetricsTracer', () => { + it('should not call _initMetricsTracer when metricsTracer key is explicitly set to undefined', () => { const spanner = new Spanner({projectId: 'test-project'}); // eslint-disable-next-line @typescript-eslint/no-explicit-any (spanner as any)._initMetricsTracer = sandbox.stub(); @@ -241,11 +240,11 @@ describe('RequestStreamCoordinator', () => { const config: any = { client: 'SpannerClient', method: 'batchWrite', - metricsTracer: null, + metricsTracer: undefined, }; spanner.requestStream(config); - assert.strictEqual(config.metricsTracer, null); + assert.strictEqual(config.metricsTracer, undefined); assert.strictEqual( // eslint-disable-next-line @typescript-eslint/no-explicit-any ((spanner as any)._initMetricsTracer as sinon.SinonStub).called, From 698b05ae4f2e532d15d86b91eb41478c3666e99d Mon Sep 17 00:00:00 2001 From: Alka Trivedi Date: Thu, 19 Nov 2026 19:05:16 +0000 Subject: [PATCH 3/3] address review comment --- handwritten/spanner/src/batch-write-stream.ts | 11 +- handwritten/spanner/src/common.ts | 10 +- handwritten/spanner/src/database.ts | 1 + .../spanner/test/batch-write-stream.ts | 279 +++++++++++++++++- handwritten/spanner/test/database.ts | 26 +- 5 files changed, 299 insertions(+), 28 deletions(-) diff --git a/handwritten/spanner/src/batch-write-stream.ts b/handwritten/spanner/src/batch-write-stream.ts index 6eba983168c8..a4a9d0bdb57c 100644 --- a/handwritten/spanner/src/batch-write-stream.ts +++ b/handwritten/spanner/src/batch-write-stream.ts @@ -54,12 +54,12 @@ const DEFAULT_TOTAL_TIMEOUT_MS = 3600000; // 1 hour */ function isRetryableError( err: grpc.ServiceError, - retryCodes?: grpc.status[] | null, + retry?: CallOptions['retry'], ): boolean { - if (!err || typeof err.code !== 'number') { + if (retry === null || !err || typeof err.code !== 'number') { return false; } - const retryableCodes = retryCodes ?? [grpc.status.UNAVAILABLE]; + const retryableCodes = retry?.retryCodes ?? [grpc.status.UNAVAILABLE]; const allowsNetworkRetries = retryableCodes.includes(grpc.status.UNAVAILABLE); return ( retryableCodes.includes(err.code) || @@ -161,8 +161,9 @@ function handleRetryOrFail( gaxOptions?.timeout ?? backoffSettings?.totalTimeoutMillis ?? DEFAULT_TOTAL_TIMEOUT_MS; - const maxRetries = backoffSettings?.maxRetries ?? Infinity; - const retryable = isRetryableError(err, gaxOptions?.retry?.retryCodes); + const maxRetries = + gaxOptions?.maxRetries ?? backoffSettings?.maxRetries ?? Infinity; + const retryable = isRetryableError(err, gaxOptions?.retry); const elapsed = Date.now() - state.startTime; if ( diff --git a/handwritten/spanner/src/common.ts b/handwritten/spanner/src/common.ts index 6135c7f6c638..e77738951a65 100644 --- a/handwritten/spanner/src/common.ts +++ b/handwritten/spanner/src/common.ts @@ -174,9 +174,9 @@ export function getNextRetryDelayMs( NonNullable['backoffSettings']> >, ): number { - const retryInfoValues = err?.metadata && err.metadata.get(RETRY_INFO_BIN); - if (retryInfoValues && retryInfoValues.length > 0) { - try { + try { + const retryInfoValues = err?.metadata?.get?.(RETRY_INFO_BIN); + if (retryInfoValues && retryInfoValues.length > 0) { // eslint-disable-next-line @typescript-eslint/no-explicit-any const {retryDelay} = (RetryInfo as any).decode(retryInfoValues[0]); if (retryDelay) { @@ -195,9 +195,9 @@ export function getNextRetryDelayMs( return delayMs; } } - } catch { - // Fall back to exponential backoff if RetryInfo decoding fails. } + } catch { + // Fall back to exponential backoff if RetryInfo extraction or decoding fails. } const initialDelay = diff --git a/handwritten/spanner/src/database.ts b/handwritten/spanner/src/database.ts index 4544e99193de..513cc1ebad9e 100644 --- a/handwritten/spanner/src/database.ts +++ b/handwritten/spanner/src/database.ts @@ -3831,6 +3831,7 @@ class Database extends common.GrpcServiceObject { }, }); delete baseGaxOpts.retry; + delete baseGaxOpts.maxRetries; const hasExplicitTimeout = options?.gaxOptions?.timeout !== undefined || options?.gaxOptions?.retry?.backoffSettings?.totalTimeoutMillis !== diff --git a/handwritten/spanner/test/batch-write-stream.ts b/handwritten/spanner/test/batch-write-stream.ts index 4a4c005c8777..518d0ac0216d 100644 --- a/handwritten/spanner/test/batch-write-stream.ts +++ b/handwritten/spanner/test/batch-write-stream.ts @@ -759,7 +759,78 @@ describe('batchWriteStream', () => { attemptStream.emit('error', eosErr); }); - it('should cancel pending retry timer and active stream when caller destroys stream early', done => { + it('should disable retries on UNAVAILABLE when gaxOptions.retry is null', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: {retry: null}, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const unavailableErr = makeServiceError( + grpc.status.UNAVAILABLE, + 'Service currently unavailable', + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err, unavailableErr); + assert.strictEqual(finalStatus, grpc.status.UNAVAILABLE); + done(); + }); + + attemptStream.emit('error', unavailableErr); + }); + + it('should disable retries on premature stream end when gaxOptions.retry is null', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: {retry: null}, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(responses.length, 1); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.strictEqual(err.code, grpc.status.UNAVAILABLE); + assert.match(err.message, /ended prematurely/); + assert.strictEqual(finalStatus, grpc.status.UNAVAILABLE); + done(); + }); + + attemptStream.write({indexes: [0], status: {code: grpc.status.OK}}); + attemptStream.end(); + }); + + it('should cancel pending retry timer and active stream when caller destroys stream early', () => { + const clock = sandbox.useFakeTimers(); const attemptStream = through.obj(); let requestCount = 0; let finalStatus: grpc.status | undefined; @@ -797,11 +868,9 @@ describe('batchWriteStream', () => { // Caller destroys stream while retry timer is waiting stream.destroy(); - setTimeout(() => { - assert.strictEqual(requestCount, 1); - assert.strictEqual(finalStatus, grpc.status.CANCELLED); - done(); - }, 80); + clock.tick(200); + assert.strictEqual(requestCount, 1); + assert.strictEqual(finalStatus, grpc.status.CANCELLED); }); it('should emit DeadlineError and finalize with DEADLINE_EXCEEDED when retry delay exceeds total timeout', done => { @@ -957,6 +1026,179 @@ describe('batchWriteStream', () => { }); }); + it('should respect top-level gaxOptions.maxRetries (taking precedence over backoffSettings.maxRetries)', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const unavailableErr = makeServiceError( + grpc.status.UNAVAILABLE, + 'Service currently unavailable', + ); + + const stream = batchWriteStream( + [group0], + () => { + const current = attemptStreams[requestCount++]; + setImmediate(() => { + current.emit('error', unavailableErr); + }); + return current; + }, + { + gaxOptions: { + maxRetries: 1, + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + maxRetries: 5, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + // 1 initial attempt + 1 retry = 2 attempts total + assert.strictEqual(requestCount, 2); + assert.strictEqual(err, unavailableErr); + assert.strictEqual(finalStatus, grpc.status.UNAVAILABLE); + done(); + }); + }); + + it('should pause active request stream on downstream backpressure and resume on drain', done => { + const totalGroups = 40; // Exceeds through2.obj readable + writable highWaterMark (16 + 16 = 32) + const groups: IMutationGroup[] = Array.from( + {length: totalGroups}, + (_, i) => ({ + mutations: [{insert: {table: `T${i}`}}], + }), + ); + const attemptStream = through.obj(); + const pauseSpy = sandbox.spy(attemptStream, 'pause'); + const resumeSpy = sandbox.spy(attemptStream, 'resume'); + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream(groups, () => attemptStream, { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }); + + const initialResumeCount = resumeSpy.callCount; + + // Push 40 responses while no consumer is reading `stream` to trigger backpressure + for (let i = 0; i < totalGroups; i++) { + attemptStream.write({indexes: [i], status: {code: grpc.status.OK}}); + } + attemptStream.end(); + + assert.strictEqual( + pauseSpy.called, + true, + 'Expected upstream request stream to be paused on backpressure', + ); + + const receivedIndexes: number[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => { + receivedIndexes.push(...(resp.indexes ?? [])); + }) + .on('error', done) + .on('end', () => { + assert.ok( + resumeSpy.callCount > initialResumeCount, + 'Expected upstream request stream to be resumed on drain', + ); + assert.deepStrictEqual( + receivedIndexes, + Array.from({length: totalGroups}, (_, i) => i), + ); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should handle downstream backpressure cleanly across stream retries', done => { + const totalGroups = 50; + const groups: IMutationGroup[] = Array.from( + {length: totalGroups}, + (_, i) => ({ + mutations: [{insert: {table: `T${i}`}}], + }), + ); + const stream1 = through.obj(); + const stream2 = through.obj(); + const stream1PauseSpy = sandbox.spy(stream1, 'pause'); + const stream2PauseSpy = sandbox.spy(stream2, 'pause'); + const stream2ResumeSpy = sandbox.spy(stream2, 'resume'); + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + groups, + currentGroups => { + attemptIndex++; + if (attemptIndex === 1) { + setImmediate(() => { + // Emit 40 responses with no downstream reader -> triggers backpressure on stream1 + for (let i = 0; i < 40; i++) { + stream1.write({indexes: [i], status: {code: grpc.status.OK}}); + } + assert.strictEqual(stream1PauseSpy.called, true); + stream1.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + }); + return stream1; + } + + setImmediate(() => { + const initialStream2Resumes = stream2ResumeSpy.callCount; + // userStream buffer is still full -> writing remaining 10 responses pauses stream2 + for (let i = 0; i < currentGroups.length; i++) { + stream2.write({indexes: [i], status: {code: grpc.status.OK}}); + } + stream2.end(); + assert.strictEqual(stream2PauseSpy.called, true); + + // Start consuming downstream so userStream drains and resumes stream2 + const receivedIndexes: number[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => { + receivedIndexes.push(...(resp.indexes ?? [])); + }) + .on('error', done) + .on('end', () => { + assert.ok(stream2ResumeSpy.callCount > initialStream2Resumes); + assert.deepStrictEqual( + receivedIndexes, + Array.from({length: totalGroups}, (_, i) => i), + ); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + return stream2; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + }); + describe('getNextRetryDelayMs', () => { it('should extract delay from google.rpc.retryinfo-bin metadata when present', () => { const RetryInfo = getRetryInfo(); @@ -1007,5 +1249,30 @@ describe('batchWriteStream', () => { `Expected delay in [80, 120], got ${delayMs}`, ); }); + + it('should fall back to jittered exponential backoff when metadata is undefined or lacks a get method', () => { + const errWithoutMetadata = Object.assign(new Error('Unavailable'), { + code: grpc.status.UNAVAILABLE, + }) as grpc.ServiceError; + const errWithPlainObjectMetadata = Object.assign( + new Error('Unavailable'), + { + code: grpc.status.UNAVAILABLE, + metadata: {} as unknown as grpc.Metadata, + }, + ) as grpc.ServiceError; + + for (const err of [errWithoutMetadata, errWithPlainObjectMetadata]) { + const delayMs = getNextRetryDelayMs(err, 1, { + initialRetryDelayMillis: 100, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 32000, + }); + assert.ok( + delayMs >= 80 && delayMs <= 120, + `Expected delay in [80, 120], got ${delayMs}`, + ); + } + }); }); }); diff --git a/handwritten/spanner/test/database.ts b/handwritten/spanner/test/database.ts index ebc24945d80c..b9641e77a6ca 100644 --- a/handwritten/spanner/test/database.ts +++ b/handwritten/spanner/test/database.ts @@ -916,7 +916,13 @@ describe('Database', () => { const stream1 = through.obj(); const stream2 = through.obj(); requestStreamStub.onCall(0).returns(stream1); - requestStreamStub.onCall(1).returns(stream2); + requestStreamStub.onCall(1).callsFake(() => { + setImmediate(() => { + stream2.write({indexes: [0], status: {code: grpc.status.OK}}); + stream2.end(); + }); + return stream2; + }); const retryOptions: BatchWriteOptions = { ...options, @@ -993,11 +999,6 @@ describe('Database', () => { code: grpc.status.UNAVAILABLE, }), ); - - setTimeout(() => { - stream2.write({indexes: [0], status: {code: grpc.status.OK}}); - stream2.end(); - }, 15); }); it('should initialize and pass metricsTracer across all retry attempts in batchWriteAtLeastOnce', done => { @@ -1011,7 +1012,13 @@ describe('Database', () => { sandbox.stub(spanner, '_initMetricsTracer').returns(fakeTracer); requestStreamStub.onCall(0).returns(stream1); - requestStreamStub.onCall(1).returns(stream2); + requestStreamStub.onCall(1).callsFake(() => { + setImmediate(() => { + stream2.write({indexes: [0], status: {code: grpc.status.OK}}); + stream2.end(); + }); + return stream2; + }); const retryOptions: BatchWriteOptions = { gaxOptions: { @@ -1055,11 +1062,6 @@ describe('Database', () => { code: grpc.status.UNAVAILABLE, }), ); - - setTimeout(() => { - stream2.write({indexes: [0], status: {code: grpc.status.OK}}); - stream2.end(); - }, 15); }); it('should not call `requestStream` or initialize metrics if stream is destroyed before `getSession` resolves', done => {