diff --git a/handwritten/spanner/observability-test/database.ts b/handwritten/spanner/observability-test/database.ts index 1b3a3e23df4b..c6253c7df095 100644 --- a/handwritten/spanner/observability-test/database.ts +++ b/handwritten/spanner/observability-test/database.ts @@ -204,6 +204,7 @@ describe('Database', () => { const SPANNER = { routeToLeaderEnabled: true, + _initMetricsTracer: util.noop, } as {} as Spanner; const INSTANCE = { @@ -1282,8 +1283,68 @@ describe('Database', () => { done(); }); - fakeDataStream.emit('data', 'response'); - fakeDataStream.end('end'); + fakeDataStream.emit('data', {indexes: [0, 1]}); + fakeDataStream.end(); + }); + + it('ends span and does not start RPC when stream is destroyed before getSession resolves', done => { + let sessionCallback: + ((err: Error | null, session?: FakeSession) => void) | null = null; + getSessionStub.callsFake(callback => { + sessionCallback = callback; + }); + + const stream = database.batchWriteAtLeastOnce(mutationGroups, options); + stream.destroy(); + + setImmediate(() => { + sessionCallback!(null, fakeSession); + const spans = traceExporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1, 'Exactly 1 span expected'); + withAllSpansHaveDBName(spans); + assert.strictEqual( + spans[0].name, + 'CloudSpanner.Database.batchWriteAtLeastOnce', + ); + assert.strictEqual(spans[0].events.length, 0); + done(); + }); + }); + + it('records error on span when stream is destroyed with an error', done => { + const abortErr = new Error('abort'); + const stream = database.batchWriteAtLeastOnce(mutationGroups, options); + stream.on('error', err => { + assert.strictEqual(err, abortErr); + const spans = traceExporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1, 'Exactly 1 span expected'); + withAllSpansHaveDBName(spans); + assert.strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + assert.strictEqual(spans[0].status.message, 'abort'); + done(); + }); + stream.destroy(abortErr); + }); + + it('ends span immediately with no events when mutationGroups is empty', done => { + database + .batchWriteAtLeastOnce([], options) + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', done) + .on('end', () => { + const spans = traceExporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1, 'Exactly 1 span expected'); + withAllSpansHaveDBName(spans); + assert.strictEqual( + spans[0].name, + 'CloudSpanner.Database.batchWriteAtLeastOnce', + ); + assert.strictEqual(spans[0].events.length, 0); + assert.notStrictEqual(spans[0].status.code, SpanStatusCode.ERROR); + done(); + }); }); }); diff --git a/handwritten/spanner/src/batch-write-stream.ts b/handwritten/spanner/src/batch-write-stream.ts new file mode 100644 index 000000000000..a4a9d0bdb57c --- /dev/null +++ b/handwritten/spanner/src/batch-write-stream.ts @@ -0,0 +1,372 @@ +/*! + * Copyright 2026 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {CallOptions, grpc} from 'google-gax'; +import {Readable, Transform} from 'stream'; +import * as through from 'through2'; +import {protos} from '@google-cloud/spanner-api'; +import {DeadlineError, isRetryableInternalError} from './transaction-runner'; +import {getNextRetryDelayMs} from './common'; + +type IMutationGroup = protos.google.spanner.v1.BatchWriteRequest.IMutationGroup; +type IBatchWriteResponse = protos.google.spanner.v1.IBatchWriteResponse; + +type MakeBatchWriteRequest = ( + currentMutationGroups: IMutationGroup[], + remainingTimeoutMs: number, +) => Readable; + +export interface BatchWriteStreamOptions { + gaxOptions?: CallOptions; + onTerminate?: (finalStatus: grpc.status) => void; +} + +interface BatchWriteState { + makeRequest: MakeBatchWriteRequest; + options: BatchWriteStreamOptions; + userStream: Transform; + pending: Array; + remaining: number; + activeRequestStream: Readable | null; + retryTimer: NodeJS.Timeout | null; + retryCount: number; + startTime: number; + terminated: boolean; +} + +const DEFAULT_TOTAL_TIMEOUT_MS = 3600000; // 1 hour + +/** + * Checks whether an error is eligible for retry. + */ +function isRetryableError( + err: grpc.ServiceError, + retry?: CallOptions['retry'], +): boolean { + if (retry === null || !err || typeof err.code !== 'number') { + return false; + } + const retryableCodes = retry?.retryCodes ?? [grpc.status.UNAVAILABLE]; + const allowsNetworkRetries = retryableCodes.includes(grpc.status.UNAVAILABLE); + return ( + retryableCodes.includes(err.code) || + (allowsNetworkRetries && isRetryableInternalError(err)) + ); +} + +/** + * Validates response indexes against current request bounds, translates + * retry-local indexes back to original caller positions in-place, and marks + * acknowledged mutation groups as resolved. + */ +function processResponseIndexes( + response: IBatchWriteResponse, + streamIndexes: number[], + pending: Array, +): {error?: grpc.ServiceError; acknowledgedCount: number} { + const respIndexes = response.indexes; + if (!Array.isArray(respIndexes)) { + return {acknowledgedCount: 0}; + } + + // Validate all response indexes before mutating state. + for (let i = 0; i < respIndexes.length; i++) { + const idx = respIndexes[i]; + if ( + typeof idx !== 'number' || + !Number.isInteger(idx) || + idx < 0 || + idx >= streamIndexes.length + ) { + const msg = `BatchWrite index ${idx} out of bounds for current request size ${streamIndexes.length}`; + return { + error: Object.assign(new Error(msg), { + code: grpc.status.INTERNAL, + details: msg, + metadata: new grpc.Metadata(), + } as grpc.ServiceError), + acknowledgedCount: 0, + }; + } + } + + // Translate retry-local indexes back to original caller indexes in-place + // and mark each acknowledged group as resolved (nulling reference for GC). + let acknowledgedCount = 0; + for (let i = 0; i < respIndexes.length; i++) { + const retryIdx = respIndexes[i]; + const origIdx = streamIndexes[retryIdx]; + respIndexes[i] = origIdx; + if (pending[origIdx] !== null) { + pending[origIdx] = null; + acknowledgedCount++; + } + } + + return {acknowledgedCount}; +} + +function finalize(state: BatchWriteState, status: grpc.status): void { + if (state.terminated) { + return; + } + state.terminated = true; + if (state.retryTimer) { + clearTimeout(state.retryTimer); + state.retryTimer = null; + } + state.options.onTerminate?.(status); +} + +function cleanupActiveStream(state: BatchWriteState): void { + if (!state.activeRequestStream) { + return; + } + const streamToClean = state.activeRequestStream; + state.activeRequestStream = null; + streamToClean.removeAllListeners('data'); + streamToClean.removeAllListeners('end'); + streamToClean.removeAllListeners('error'); + streamToClean.on('error', () => {}); + if (!streamToClean.destroyed) { + streamToClean.destroy(); + } +} + +function handleRetryOrFail( + state: BatchWriteState, + err: grpc.ServiceError, +): void { + cleanupActiveStream(state); + if (state.terminated || state.userStream.destroyed) { + return; + } + + const gaxOptions = state.options.gaxOptions; + const backoffSettings = gaxOptions?.retry?.backoffSettings; + const timeout = + gaxOptions?.timeout ?? + backoffSettings?.totalTimeoutMillis ?? + DEFAULT_TOTAL_TIMEOUT_MS; + const maxRetries = + gaxOptions?.maxRetries ?? backoffSettings?.maxRetries ?? Infinity; + const retryable = isRetryableError(err, gaxOptions?.retry); + const elapsed = Date.now() - state.startTime; + + if ( + state.remaining > 0 && + state.retryCount < maxRetries && + elapsed < timeout && + retryable + ) { + state.retryCount++; + const delayMs = getNextRetryDelayMs(err, state.retryCount, backoffSettings); + if (elapsed + delayMs >= timeout) { + const deadlineErr = new DeadlineError(err); + finalize(state, grpc.status.DEADLINE_EXCEEDED); + state.userStream.destroy(deadlineErr); + return; + } + state.retryTimer = setTimeout(() => { + state.retryTimer = null; + if (!state.terminated && !state.userStream.destroyed) { + startAttempt(state); + } + }, delayMs); + return; + } + + const finalError = + elapsed >= timeout && retryable ? new DeadlineError(err) : err; + finalize(state, finalError.code ?? grpc.status.UNKNOWN); + state.userStream.destroy(finalError); +} + +function startAttempt(state: BatchWriteState): void { + if (state.terminated || state.userStream.destroyed) { + return; + } + + if (state.remaining === 0) { + finalize(state, grpc.status.OK); + state.userStream.end(); + return; + } + + // Filter unacknowledged mutation groups and record index mapping + const currentGroups: IMutationGroup[] = []; + const streamIndexes: number[] = []; + for (let origIdx = 0; origIdx < state.pending.length; origIdx++) { + const group = state.pending[origIdx]; + if (group !== null) { + currentGroups.push(group); + streamIndexes.push(origIdx); + } + } + + const gaxOptions = state.options.gaxOptions; + const timeout = + gaxOptions?.timeout ?? + gaxOptions?.retry?.backoffSettings?.totalTimeoutMillis ?? + DEFAULT_TOTAL_TIMEOUT_MS; + const elapsed = Date.now() - state.startTime; + const remainingTimeoutMs = Number.isFinite(timeout) + ? Math.max(0, timeout - elapsed) + : timeout; + + const currentStream = state.makeRequest(currentGroups, remainingTimeoutMs); + state.activeRequestStream = currentStream; + + currentStream.on('data', (response: IBatchWriteResponse) => { + if (state.terminated || state.userStream.destroyed) { + return; + } + + const {error, acknowledgedCount} = processResponseIndexes( + response, + streamIndexes, + state.pending, + ); + + if (error) { + cleanupActiveStream(state); + finalize(state, grpc.status.INTERNAL); + state.userStream.destroy(error); + return; + } + + state.remaining -= acknowledgedCount; + + // Forward response and handle downstream backpressure + if (!state.userStream.write(response)) { + currentStream.pause(); + } + }); + + currentStream.once('error', (err: grpc.ServiceError) => { + if (state.terminated || state.userStream.destroyed) { + return; + } + if (state.pending.length > 0 && state.remaining === 0) { + cleanupActiveStream(state); + finalize(state, grpc.status.OK); + state.userStream.end(); + return; + } + handleRetryOrFail(state, err); + }); + + currentStream.once('end', () => { + cleanupActiveStream(state); + if (state.terminated || state.userStream.destroyed) { + return; + } + + // Premature stream end (EOF while remaining > 0) is treated as a + // retryable UNAVAILABLE error so remaining groups are retried. + if (state.remaining > 0) { + const msg = `BatchWrite stream ended prematurely with ${state.remaining} unacknowledged mutation group(s)`; + const prematureEofErr = Object.assign(new Error(msg), { + code: grpc.status.UNAVAILABLE, + details: msg, + metadata: new grpc.Metadata(), + } as grpc.ServiceError); + handleRetryOrFail(state, prematureEofErr); + return; + } + + finalize(state, grpc.status.OK); + state.userStream.end(); + }); +} + +/** + * Creates a readable object stream that executes `BatchWrite` and transparently + * retries and resumes on transient stream failures (`UNAVAILABLE`, retryable `INTERNAL`, + * and premature stream `end`) for only the unresolved mutation groups. + */ +export function batchWriteStream( + mutationGroups: IMutationGroup[], + makeRequest: MakeBatchWriteRequest, + options: BatchWriteStreamOptions = {}, +): Transform { + // Track pending mutation groups by their original caller index [0..N-1]. + // Resolved entries are set to `null` so large mutation payloads can be GC'd. + const pending: Array = mutationGroups + ? [...mutationGroups] + : []; + + const userStream: Transform = through.obj( + (chunk: IBatchWriteResponse, _enc, next) => { + next(null, chunk); + }, + ); + + const state: BatchWriteState = { + makeRequest, + options, + userStream, + pending, + remaining: pending.length, + activeRequestStream: null, + retryTimer: null, + retryCount: 0, + startTime: Date.now(), + terminated: false, + }; + + // Hook caller cancellation and cleanup + const originalDestroy = userStream._destroy; + userStream._destroy = ( + error: Error | null, + callback: (error?: Error | null) => void, + ) => { + if (!state.terminated) { + cleanupActiveStream(state); + finalize( + state, + (error as grpc.ServiceError | null)?.code ?? grpc.status.CANCELLED, + ); + } + if (typeof originalDestroy === 'function') { + originalDestroy.call(userStream, error, callback); + } else { + callback(error); + } + }; + + userStream.once('close', () => { + if (!state.terminated) { + cleanupActiveStream(state); + finalize(state, grpc.status.CANCELLED); + } + }); + + userStream.on('drain', () => { + state.activeRequestStream?.resume(); + }); + + if (pending.length === 0) { + finalize(state, grpc.status.OK); + setImmediate(() => { + userStream.end(); + }); + return userStream; + } + + startAttempt(state); + return userStream; +} diff --git a/handwritten/spanner/src/common.ts b/handwritten/spanner/src/common.ts index 3dc1be2ed870..e77738951a65 100644 --- a/handwritten/spanner/src/common.ts +++ b/handwritten/spanner/src/common.ts @@ -16,6 +16,10 @@ import {grpc, CallOptions, Operation as GaxOperation} from 'google-gax'; import {protos} from '@google-cloud/spanner-api'; +import {getRetryInfo, RETRY_INFO_BIN} from './protos'; + +export {RETRY_INFO_BIN}; +const RetryInfo = getRetryInfo(); import instanceAdmin = protos.google; import databaseAdmin = protos.google; @@ -153,3 +157,61 @@ export function getCommonHeaders( return headers; } + +// Default retry/backoff parameters from spanner_client_config.json +const DEFAULT_INITIAL_RETRY_DELAY_MS = 100; +const DEFAULT_MAX_RETRY_DELAY_MS = 60000; +const DEFAULT_RETRY_DELAY_MULTIPLIER = 1.3; + +/** + * Extracts server-supplied retry delay (in ms) from `google.rpc.retryinfo-bin` + * trailing metadata if present, otherwise computes jittered exponential backoff. + */ +export function getNextRetryDelayMs( + err: grpc.ServiceError, + retryAttempt: number, + backoffSettings?: Partial< + NonNullable['backoffSettings']> + >, +): number { + try { + const retryInfoValues = err?.metadata?.get?.(RETRY_INFO_BIN); + if (retryInfoValues && retryInfoValues.length > 0) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const {retryDelay} = (RetryInfo as any).decode(retryInfoValues[0]); + if (retryDelay) { + let {seconds} = retryDelay; + if ( + seconds && + typeof seconds !== 'number' && + typeof seconds.toNumber === 'function' + ) { + seconds = seconds.toNumber(); + } + const secondsInMs = Math.floor(Number(seconds || 0)) * 1000; + const nanosInMs = Math.floor(Number(retryDelay.nanos || 0)) / 1e6; + const delayMs = secondsInMs + nanosInMs; + if (delayMs >= 0) { + return delayMs; + } + } + } + } catch { + // Fall back to exponential backoff if RetryInfo extraction or decoding fails. + } + + const initialDelay = + backoffSettings?.initialRetryDelayMillis ?? DEFAULT_INITIAL_RETRY_DELAY_MS; + const maxDelay = + backoffSettings?.maxRetryDelayMillis ?? DEFAULT_MAX_RETRY_DELAY_MS; + const multiplier = + backoffSettings?.retryDelayMultiplier ?? DEFAULT_RETRY_DELAY_MULTIPLIER; + + const unclampedDelay = + initialDelay * Math.pow(multiplier, Math.max(0, retryAttempt - 1)); + const cappedDelay = Math.min(maxDelay, unclampedDelay); + + // Jitter in [0.8 * cappedDelay, 1.2 * cappedDelay] + const jitterFactor = 0.8 + Math.random() * 0.4; + return Math.floor(cappedDelay * jitterFactor); +} diff --git a/handwritten/spanner/src/database.ts b/handwritten/spanner/src/database.ts index a239358bc959..513cc1ebad9e 100644 --- a/handwritten/spanner/src/database.ts +++ b/handwritten/spanner/src/database.ts @@ -38,6 +38,7 @@ import { } from 'google-gax'; import {Backup} from './backup'; import {BatchTransaction, TransactionIdentifier} from './batch-transaction'; +import {batchWriteStream} from './batch-write-stream'; import { GetSessionCallback, SessionFactory, @@ -192,6 +193,19 @@ type PoolRequestCallback = RequestCallback; type ResultSetStats = spannerClient.spanner.v1.ResultSetStats; type ResultSetMetadata = spannerClient.spanner.v1.ResultSetMetadata; +type IMutationGroup = spannerClient.spanner.v1.BatchWriteRequest.IMutationGroup; + +interface BatchWriteAttemptParams { + session: Session; + currentMutationGroups: IMutationGroup[]; + remainingTimeoutMs: number; + attempt: number; + nthRequest: number; + baseGaxOpts: CallOptions; + hasExplicitTimeout: boolean; + options?: BatchWriteOptions; + metricsTracer?: RequestConfig['metricsTracer']; +} export type GetSessionsOptions = PagedOptionsWithFilter; export type GetDatabaseRolesOptions = PagedOptionsWithFilter; @@ -3594,6 +3608,13 @@ class Database extends common.GrpcServiceObject { * rows being added to the mutation's table. We recommend structuring your mutation groups to * be idempotent to avoid this issue. * + * If the stream is interrupted by a retryable error (such as `UNAVAILABLE`) or closes + * prematurely before all mutation groups have been acknowledged, the client automatically + * retries and resumes the request for only the remaining unacknowledged mutation groups. + * The `indexes` field in each emitted {@link protos.google.spanner.v1.BatchWriteResponse|BatchWriteResponse} + * always corresponds to the original positions in the caller's `mutationGroups` array, + * even across retries. + * * @method Spanner#batchWriteAtLeastOnce * * @param {MutationGroup[]} [mutationGroups] The group of mutations to be applied. @@ -3649,45 +3670,48 @@ class Database extends common.GrpcServiceObject { transactionTag: options?.requestOptions?.transactionTag, }, span => { + let dataStream: Transform | null = null; + const endSpan = this._bindBatchWriteStreamToSpan( + proxyStream, + span, + () => dataStream, + ); + + if (!mutationGroups || mutationGroups.length === 0) { + setImmediate(() => { + endSpan(); + proxyStream.end(); + }); + return proxyStream as NodeJS.ReadableStream; + } + this.sessionFactory_.getSession((err, session) => { + if (proxyStream.destroyed) { + endSpan(err); + return; + } + if (err) { + endSpan(err); proxyStream.destroy(err); - setSpanError(span, err); - span.end(); return; } span.addEvent('Using Session', {'session.id': session?.id}); - const gaxOpts = extend(true, {}, options?.gaxOptions); - const reqOpts: spannerClient.spanner.v1.IBatchWriteRequest = { - session: session!.formattedName_!, - mutationGroups: mutationGroups.map(mg => mg.proto()), - requestOptions: options?.requestOptions, - excludeTxnFromChangeStreams: options?.excludeTxnFromChangeStreams, - }; - const headers = this._metadataWithRequestId( - this._nextNthRequest(), - 1, - this.commonHeaders_, + dataStream = this._createBatchWriteStream( + session!, + mutationGroups, + options, + span, ); - if (this._getSpanner().routeToLeaderEnabled) { - addLeaderAwareRoutingHeader(headers); - } - const dataStream = this.requestStream({ - client: 'SpannerClient', - method: 'batchWrite', - reqOpts, - gaxOpts, - headers, - }); + dataStream - .once('error', err => { - setSpanError(span, err); - span.end(); - proxyStream.destroy(err); + .once('error', streamErr => { + endSpan(streamErr); + proxyStream.destroy(streamErr); }) .once('end', () => { - span.end(); + endSpan(); }) .pipe(proxyStream); }); @@ -3697,6 +3721,168 @@ class Database extends common.GrpcServiceObject { ); } + /** + * Attaches destruction and close listeners to the user-facing proxy stream + * to ensure the trace span ends and the active gRPC stream is destroyed on early abort. + */ + private _bindBatchWriteStreamToSpan( + proxyStream: Transform, + span: Span, + getDataStream: () => Transform | null, + ): (err?: Error | null) => void { + let spanEnded = false; + const endSpan = (err?: Error | null) => { + if (spanEnded) { + return; + } + spanEnded = true; + if (err) { + setSpanError(span, err); + } + span.end(); + }; + + const originalProxyDestroy = proxyStream._destroy; + proxyStream._destroy = function ( + destroyErr: Error | null, + callback: (error?: Error | null) => void, + ) { + endSpan(destroyErr); + const dataStream = getDataStream(); + if (dataStream && !dataStream.destroyed) { + dataStream.destroy(destroyErr || undefined); + } + if (typeof originalProxyDestroy === 'function') { + originalProxyDestroy.call(this, destroyErr, callback); + } else { + callback(destroyErr); + } + }; + + proxyStream.once('close', () => { + endSpan(); + const dataStream = getDataStream(); + if (dataStream && !dataStream.destroyed) { + dataStream.destroy(); + } + }); + + return endSpan; + } + + /** + * Issues an individual BatchWrite attempt RPC with appropriate timeout, + * request headers, and metrics tracing. + */ + private _createBatchWriteAttempt({ + session, + currentMutationGroups, + remainingTimeoutMs, + attempt, + nthRequest, + baseGaxOpts, + hasExplicitTimeout, + options, + metricsTracer, + }: BatchWriteAttemptParams): Readable { + const reqOpts: spannerClient.spanner.v1.IBatchWriteRequest = { + session: session.formattedName_!, + mutationGroups: currentMutationGroups, + requestOptions: options?.requestOptions, + excludeTxnFromChangeStreams: options?.excludeTxnFromChangeStreams, + }; + const headers = this._metadataWithRequestId( + nthRequest, + attempt, + this.commonHeaders_, + ); + if (this._getSpanner()?.routeToLeaderEnabled) { + addLeaderAwareRoutingHeader(headers); + } + const attemptGaxOpts = + (hasExplicitTimeout || attempt > 1) && Number.isFinite(remainingTimeoutMs) + ? Object.assign({}, baseGaxOpts, {timeout: remainingTimeoutMs}) + : baseGaxOpts; + const requestConfig: RequestConfig = { + client: 'SpannerClient', + method: 'batchWrite', + reqOpts, + gaxOpts: attemptGaxOpts, + headers, + metricsTracer, + }; + return this.requestStream(requestConfig); + } + + /** + * Creates the resilient BatchWrite stream for an active session, coordinating + * retries across attempts and recording final metrics upon termination. + */ + private _createBatchWriteStream( + session: Session, + mutationGroups: MutationGroup[], + options: BatchWriteOptions | undefined, + span: Span, + ): Transform { + const baseGaxOpts = extend(true, {}, options?.gaxOptions, { + retryRequestOptions: { + noResponseRetries: 0, + retries: 0, + }, + }); + delete baseGaxOpts.retry; + delete baseGaxOpts.maxRetries; + const hasExplicitTimeout = + options?.gaxOptions?.timeout !== undefined || + options?.gaxOptions?.retry?.backoffSettings?.totalTimeoutMillis !== + undefined; + const mutationGroupProtos = (mutationGroups ?? []).map(mg => mg.proto()); + const nthRequest = this._nextNthRequest(); + let attempt = 0; + + const initialHeaders = this._metadataWithRequestId( + nthRequest, + 1, + this.commonHeaders_, + ); + const metricsTracer = this._initMetricsTracer({ + client: 'SpannerClient', + method: 'batchWrite', + reqOpts: {session: session.formattedName_!}, + headers: initialHeaders, + }); + + return batchWriteStream( + mutationGroupProtos, + (currentMutationGroups, remainingTimeoutMs) => { + attempt++; + if (attempt > 1) { + span.addEvent('Resuming BatchWrite stream', { + attempt, + remaining_mutation_groups: currentMutationGroups.length, + }); + } + return this._createBatchWriteAttempt({ + session, + currentMutationGroups, + remainingTimeoutMs, + attempt, + nthRequest, + baseGaxOpts, + hasExplicitTimeout, + options, + metricsTracer, + }); + }, + { + gaxOptions: options?.gaxOptions, + onTerminate: finalStatus => { + metricsTracer?.recordOperationCompletion(finalStatus); + }, + }, + ); + } + /** * Write mutations using a single RPC invocation without replay protection. * @@ -4106,6 +4292,18 @@ class Database extends common.GrpcServiceObject { private _getSpanner(): Spanner { return this.instance.parent as Spanner; } + + private _initMetricsTracer( + config: RequestConfig, + ): RequestConfig['metricsTracer'] { + const spanner = this._getSpanner(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (typeof (spanner as any)?._initMetricsTracer === 'function') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (spanner as any)._initMetricsTracer(config) ?? undefined; + } + return undefined; + } } /*! Developer Documentation diff --git a/handwritten/spanner/src/index.ts b/handwritten/spanner/src/index.ts index cf33b3c5af73..28102ba00d86 100644 --- a/handwritten/spanner/src/index.ts +++ b/handwritten/spanner/src/index.ts @@ -2135,9 +2135,19 @@ class Spanner extends GrpcService { */ // eslint-disable-next-line @typescript-eslint/no-explicit-any requestStream(config): any { - const metricsTracer = this._initMetricsTracer(config); + const isExternalTracer = + config !== null && + config !== undefined && + Object.prototype.hasOwnProperty.call(config, 'metricsTracer'); + const metricsTracer = isExternalTracer + ? (config.metricsTracer ?? null) + : this._initMetricsTracer(config); const stream = streamEvents(through.obj()); - const coordinator = new RequestStreamCoordinator(stream, metricsTracer); + const coordinator = new RequestStreamCoordinator( + stream, + metricsTracer, + isExternalTracer, + ); coordinator.setup(); stream.once('reading', () => { diff --git a/handwritten/spanner/src/metrics/metrics-tracer.ts b/handwritten/spanner/src/metrics/metrics-tracer.ts index 15cf2f34f8e1..ea382960212a 100644 --- a/handwritten/spanner/src/metrics/metrics-tracer.ts +++ b/handwritten/spanner/src/metrics/metrics-tracer.ts @@ -33,10 +33,12 @@ import {isAFEServerTimingEnabled} from '../common'; class MetricAttemptTracer { private _startTime: number; public status: string; + public completed: boolean; constructor() { this._startTime = performance.now(); this.status = Status[Status.UNKNOWN]; + this.completed = false; } /** @@ -327,8 +329,15 @@ export class MetricsTracer { * @param status The status code of the attempt (default: Status.OK). */ public recordAttemptCompletion(statusCode: Status = Status.OK) { - if (!this.enabled || !this.currentOperation?.currentAttempt) return; + if ( + !this.enabled || + !this.currentOperation?.currentAttempt || + this.currentOperation.currentAttempt.completed + ) { + return; + } const currentAttempt = this.currentOperation.currentAttempt; + currentAttempt.completed = true; currentAttempt.status = Status[statusCode] ?? Status[Status.UNKNOWN]; const attemptAttributes = this._createAttemptOtelAttributes(); const endTime = performance.now(); @@ -358,11 +367,22 @@ export class MetricsTracer { /** * Records the completion of the current operation, including its status, * latency, and attempt count. Also clears the current tracer from the factory. + * @param finalStatus Optional final status code override for the operation. */ - public recordOperationCompletion() { + public recordOperationCompletion(finalStatus?: Status) { if (!this.enabled || !this.currentOperation) return; + if ( + finalStatus !== undefined && + this.currentOperation.currentAttempt && + !this.currentOperation.currentAttempt.completed + ) { + this.recordAttemptCompletion(finalStatus); + } const endTime = performance.now(); - const operationAttributes = this._createOperationOtelAttributes(); + const operationAttributes = + finalStatus !== undefined + ? this._getAttributesForStatus(finalStatus) + : this._createOperationOtelAttributes(); const operationLatencyMilliseconds = this._getMillisecondTimeDifference( this.currentOperation.startTime, endTime, diff --git a/handwritten/spanner/src/protos.ts b/handwritten/spanner/src/protos.ts index 79cfd2f3cc3f..bcca30f96eed 100644 --- a/handwritten/spanner/src/protos.ts +++ b/handwritten/spanner/src/protos.ts @@ -25,6 +25,11 @@ const RetryInfo = Root.fromJSON(statusProtoJson).lookupType( 'google.rpc.RetryInfo', ); +/** + * Binary metadata key used by gRPC to transmit RetryInfo. + */ +export const RETRY_INFO_BIN = 'google.rpc.retryinfo-bin'; + /** * Builds the filesystem path to a particular proto sub-path, for protos * stored in `@google-cloud/spanner-api`. diff --git a/handwritten/spanner/src/request-stream-coordinator.ts b/handwritten/spanner/src/request-stream-coordinator.ts index 4254babee75a..ca88ac909751 100644 --- a/handwritten/spanner/src/request-stream-coordinator.ts +++ b/handwritten/spanner/src/request-stream-coordinator.ts @@ -26,16 +26,23 @@ import {MetricsTracer} from './metrics/metrics-tracer'; export class RequestStreamCoordinator { private readonly stream: Transform; private readonly metricsTracer: MetricsTracer | null; + private readonly isExternalTracer: boolean; private callStream: CancellableStream | null = null; private statusReceived = false; + private callStreamErrored = false; private streamFinished = false; private operationCompleted = false; private requestStarted = false; private streamClosed = false; - constructor(stream: Transform, metricsTracer: MetricsTracer | null) { + constructor( + stream: Transform, + metricsTracer: MetricsTracer | null, + isExternalTracer = false, + ) { this.stream = stream; this.metricsTracer = metricsTracer; + this.isExternalTracer = isExternalTracer; } /** @@ -100,14 +107,21 @@ export class RequestStreamCoordinator { // This guard preserves OK status for normally drained streams as well as queries // using the PartialResultSet.last optimization (where the user stream completes early // while remaining trailers drain in the background). - if (this.streamFinished || this.statusReceived) { + if (this.streamFinished || this.statusReceived || this.callStreamErrored) { return; } if (this.callStream) { // In google-gax, callStream is a StreamProxy wrapping the underlying gRPC call. // Calling .destroy() on StreamProxy alone does not cancel the active gRPC call; // .cancel() must be invoked explicitly to abort the RPC on the client. - if (typeof this.callStream.cancel === 'function') { + // Guard against StreamProxy wrapping a PassThrough stream (e.g. when client is closed) + // which does not implement .cancel(). + const innerStream = (this.callStream as {stream?: {cancel?: unknown}}) + .stream; + if ( + typeof this.callStream.cancel === 'function' && + (!innerStream || typeof innerStream.cancel === 'function') + ) { this.callStream.cancel(); } // Clean up proxy and stream resources once cancellation has been requested. @@ -162,6 +176,7 @@ export class RequestStreamCoordinator { }); callStream .on('error', (error: Error) => { + this.callStreamErrored = true; if (!this.stream.destroyed && !this.streamClosed) { this.stream.destroy(error); } @@ -221,7 +236,7 @@ export class RequestStreamCoordinator { * Completes the OpenTelemetry operation metric. Idempotent. */ completeOperation(): void { - if (this.operationCompleted) { + if (this.operationCompleted || this.isExternalTracer) { return; } this.operationCompleted = true; diff --git a/handwritten/spanner/src/transaction-runner.ts b/handwritten/spanner/src/transaction-runner.ts index b1921eedb0f2..c9e721bf9d55 100644 --- a/handwritten/spanner/src/transaction-runner.ts +++ b/handwritten/spanner/src/transaction-runner.ts @@ -27,9 +27,7 @@ import google = protos.google; import IRequestOptions = google.spanner.v1.IRequestOptions; import IsolationLevel = google.spanner.v1.TransactionOptions.IsolationLevel; import ReadLockMode = google.spanner.v1.TransactionOptions.ReadWrite.ReadLockMode; -import {getRetryInfo} from './protos'; - -const RETRY_INFO = 'google.rpc.retryinfo-bin'; +import {getRetryInfo, RETRY_INFO_BIN} from './protos'; const RETRYABLE: grpc.status[] = [grpc.status.ABORTED]; @@ -157,7 +155,7 @@ export abstract class Runner { * @returns {number} Delay in milliseconds. */ getNextDelay(err: grpc.ServiceError): number { - const retryInfo = err.metadata && err.metadata.get(RETRY_INFO); + const retryInfo = err.metadata && err.metadata.get(RETRY_INFO_BIN); if (retryInfo && retryInfo.length) { // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/handwritten/spanner/src/transaction.ts b/handwritten/spanner/src/transaction.ts index 6e546f62bf14..e62e55a321d6 100644 --- a/handwritten/spanner/src/transaction.ts +++ b/handwritten/spanner/src/transaction.ts @@ -67,12 +67,12 @@ import { X_GOOG_SPANNER_REQUEST_ID_HEADER, X_GOOG_SPANNER_REQUEST_ID_SPAN_ATTR, } from './request_id_header'; +import {RETRY_INFO_BIN} from './protos'; export type Rows = Array; type ResultStats = ResultSetStats | IResultSetStats; type ResultMetadata = ResultSetMetadata | IResultSetMetadata; const RETRY_INFO_TYPE = 'type.googleapis.com/google.rpc.retryinfo'; -const RETRY_INFO_BIN = 'google.rpc.retryinfo-bin'; let nextAffinityId = 0; diff --git a/handwritten/spanner/test/batch-write-stream.ts b/handwritten/spanner/test/batch-write-stream.ts new file mode 100644 index 000000000000..518d0ac0216d --- /dev/null +++ b/handwritten/spanner/test/batch-write-stream.ts @@ -0,0 +1,1278 @@ +/*! + * Copyright 2026 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import {beforeEach, afterEach, describe, it} from 'mocha'; +import * as sinon from 'sinon'; +import * as through from 'through2'; +import {Transform} from 'stream'; +import {grpc} from 'google-gax'; +import {protos} from '@google-cloud/spanner-api'; +import {batchWriteStream} from '../src/batch-write-stream'; +import {getNextRetryDelayMs} from '../src/common'; +import {getRetryInfo, RETRY_INFO_BIN} from '../src/protos'; +import {DeadlineError} from '../src/transaction-runner'; + +type IMutationGroup = protos.google.spanner.v1.BatchWriteRequest.IMutationGroup; +type IBatchWriteResponse = protos.google.spanner.v1.IBatchWriteResponse; + +function makeServiceError( + code: grpc.status, + message: string, + metadata?: grpc.Metadata, +): grpc.ServiceError { + return Object.assign(new Error(message), { + code, + details: message, + metadata: metadata ?? new grpc.Metadata(), + }); +} + +describe('batchWriteStream', () => { + const sandbox = sinon.createSandbox(); + + const group0: IMutationGroup = {mutations: [{insert: {table: 'T0'}}]}; + const group1: IMutationGroup = {mutations: [{insert: {table: 'T1'}}]}; + const group2: IMutationGroup = {mutations: [{insert: {table: 'T2'}}]}; + const group3: IMutationGroup = {mutations: [{insert: {table: 'T3'}}]}; + const group4: IMutationGroup = {mutations: [{insert: {table: 'T4'}}]}; + + const fastGaxOptions = { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + }, + }, + }; + + beforeEach(() => {}); + + afterEach(() => { + sandbox.restore(); + }); + + it('should immediately end stream with OK and zero requests when mutationGroups is empty', done => { + let makeRequestCalled = false; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [], + () => { + makeRequestCalled = true; + return through.obj(); + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(makeRequestCalled, false); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should stream all responses and complete with OK on a single attempt', done => { + const attemptStream = through.obj(); + const receivedRequests: IMutationGroup[][] = []; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 1); + assert.deepStrictEqual(receivedRequests[0], [group0, group1]); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + + attemptStream.write({indexes: [0], status: {code: grpc.status.OK}}); + attemptStream.write({indexes: [1], status: {code: grpc.status.OK}}); + attemptStream.end(); + }); + + it('should retry only unacknowledged mutation groups on UNAVAILABLE and translate indexes across multiple retries', done => { + const attemptStreams: Transform[] = [ + through.obj(), + through.obj(), + through.obj(), + ]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1, group2, group3, group4], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Attempt 1 acknowledges original indexes 0 and 3, then fails with UNAVAILABLE + current.write({indexes: [0, 3], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else if (attemptIndex === 2) { + // Attempt 2 receives [group1, group2, group4] (retry indexes 0, 1, 2 -> original 1, 2, 4) + // Acknowledges retry index 1 (original index 2), then fails with UNAVAILABLE + current.write({indexes: [1], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else if (attemptIndex === 3) { + // Attempt 3 receives [group1, group4] (retry indexes 0, 1 -> original 1, 4) + // Acknowledges retry indexes 0 and 1 + current.write({indexes: [0, 1], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 3); + assert.deepStrictEqual(receivedRequests[0], [ + group0, + group1, + group2, + group3, + group4, + ]); + assert.deepStrictEqual(receivedRequests[1], [group1, group2, group4]); + assert.deepStrictEqual(receivedRequests[2], [group1, group4]); + + assert.strictEqual(responses.length, 3); + assert.deepStrictEqual(responses[0].indexes, [0, 3]); + assert.deepStrictEqual(responses[1].indexes, [2]); + assert.deepStrictEqual(responses[2].indexes, [1, 4]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should not re-send mutation groups that received a non-OK per-group status in BatchWriteResponse', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + + const stream = batchWriteStream( + [group0, group1, group2], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Group 0 fails with per-group ALREADY_EXISTS, Group 1 succeeds with OK + current.write({ + indexes: [0], + status: { + code: grpc.status.ALREADY_EXISTS, + message: 'Row already exists', + }, + }); + current.write({ + indexes: [1], + status: {code: grpc.status.OK}, + }); + // Stream then breaks with UNAVAILABLE before Group 2 is acknowledged + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Stream broken'), + ); + } else { + // Only Group 2 should be sent in Attempt 2 + current.write({ + indexes: [0], + status: {code: grpc.status.OK}, + }); + current.end(); + } + }); + return current; + }, + {gaxOptions: fastGaxOptions}, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[0], [group0, group1, group2]); + assert.deepStrictEqual(receivedRequests[1], [group2]); + assert.strictEqual(responses.length, 3); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.strictEqual( + responses[0].status?.code, + grpc.status.ALREADY_EXISTS, + ); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.deepStrictEqual(responses[2].indexes, [2]); + done(); + }); + }); + + it('should retry on retryable INTERNAL errors (e.g. RST_STREAM or EOS on DATA frame)', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError( + grpc.status.INTERNAL, + 'Received unexpected EOS on DATA frame from server', + ), + ); + } else { + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + {gaxOptions: fastGaxOptions}, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[1], [group1]); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + done(); + }); + }); + + it('should treat premature stream end (with unacknowledged mutation groups) as retryable UNAVAILABLE', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Only acknowledges group0, then closes stream prematurely without error + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } else { + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + {gaxOptions: fastGaxOptions}, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[1], [group1]); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + done(); + }); + }); + + it('should detach error listener on premature stream end so late errors do not trigger duplicate retries', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Acknowledge group0, end prematurely, then emit late errors on the closed stream + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + setImmediate(() => { + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Late disconnect'), + ); + current.emit( + 'error', + makeServiceError(grpc.status.CANCELLED, 'Trailing cancel'), + ); + }); + } else { + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[1], [group1]); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should finalize with OK and end stream when trailing gRPC error occurs after all mutation groups are acknowledged', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + setImmediate(() => { + attemptStream.write({indexes: [0], status: {code: grpc.status.OK}}); + attemptStream.write({indexes: [1], status: {code: grpc.status.OK}}); + attemptStream.emit( + 'error', + makeServiceError( + grpc.status.UNAVAILABLE, + 'Connection reset by peer', + ), + ); + }); + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(responses.length, 2); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should fail with INTERNAL error and not retry when server returns an out-of-bounds index', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err.code, grpc.status.INTERNAL); + assert.match(err.message, /out of bounds/); + assert.strictEqual(finalStatus, grpc.status.INTERNAL); + done(); + }); + + attemptStream.write({indexes: [2], status: {code: grpc.status.OK}}); + }); + + it('should fail with INTERNAL error and not retry when server returns an out-of-bounds index on a resumed stream', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1, group2], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Acknowledge group0, then fail with UNAVAILABLE + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else { + // Resumed stream only sent 2 groups ([group1, group2]), so index 2 is out of bounds + current.write({indexes: [2], status: {code: grpc.status.OK}}); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', (err: grpc.ServiceError) => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[0], [group0, group1, group2]); + assert.deepStrictEqual(receivedRequests[1], [group1, group2]); + assert.strictEqual(responses.length, 1); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.strictEqual(err.code, grpc.status.INTERNAL); + assert.match(err.message, /out of bounds/); + assert.strictEqual(finalStatus, grpc.status.INTERNAL); + done(); + }); + }); + + for (const invalidIndex of [-1, 1.5, null as unknown as number]) { + it(`should fail with INTERNAL error and not retry on invalid index (${String(invalidIndex)})`, done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err.code, grpc.status.INTERNAL); + assert.match(err.message, /out of bounds/); + assert.strictEqual(finalStatus, grpc.status.INTERNAL); + done(); + }); + + attemptStream.write({ + indexes: [invalidIndex], + status: {code: grpc.status.OK}, + }); + }); + } + + it('should translate duplicate indexes in-place and decrement remaining only once per unique group', done => { + const attemptStreams: Transform[] = [ + through.obj(), + through.obj(), + through.obj(), + ]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1, group2], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Acknowledge group0, then fail with UNAVAILABLE + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else if (attemptIndex === 2) { + // Resumed stream has [group1, group2] (2 groups). + // Server sends duplicate index [0, 0] for group1 and ends prematurely. + // Must NOT decrement remaining to 0 (group2 is still unacknowledged). + current.write({indexes: [0, 0], status: {code: grpc.status.OK}}); + current.end(); + } else { + // Attempt 3 must resume with only [group2] + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.end(); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', done) + .on('end', () => { + assert.strictEqual(receivedRequests.length, 3); + assert.deepStrictEqual(receivedRequests[0], [group0, group1, group2]); + assert.deepStrictEqual(receivedRequests[1], [group1, group2]); + assert.deepStrictEqual(receivedRequests[2], [group2]); + assert.strictEqual(responses.length, 3); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1, 1]); + assert.deepStrictEqual(responses[2].indexes, [2]); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should fail immediately without retrying on non-retryable RPC errors', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const permDeniedErr = makeServiceError( + grpc.status.PERMISSION_DENIED, + 'Caller lacks permission', + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err, permDeniedErr); + assert.strictEqual(finalStatus, grpc.status.PERMISSION_DENIED); + done(); + }); + + attemptStream.emit('error', permDeniedErr); + }); + + it('should emit non-retryable error and stop retrying after partial retry progress', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + const receivedRequests: IMutationGroup[][] = []; + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const permDeniedErr = makeServiceError( + grpc.status.PERMISSION_DENIED, + 'Caller lacks permission on table T1', + ); + + const stream = batchWriteStream( + [group0, group1], + groups => { + receivedRequests.push(groups); + const current = attemptStreams[attemptIndex++]; + setImmediate(() => { + if (attemptIndex === 1) { + // Attempt 1 acknowledges group0, then fails with UNAVAILABLE + current.write({indexes: [0], status: {code: grpc.status.OK}}); + current.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + } else { + // Attempt 2 receives only [group1] and fails with non-retryable PERMISSION_DENIED + current.emit('error', permDeniedErr); + } + }); + return current; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', (err: grpc.ServiceError) => { + assert.strictEqual(receivedRequests.length, 2); + assert.deepStrictEqual(receivedRequests[0], [group0, group1]); + assert.deepStrictEqual(receivedRequests[1], [group1]); + assert.strictEqual(responses.length, 1); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.strictEqual(err, permDeniedErr); + assert.strictEqual(finalStatus, grpc.status.PERMISSION_DENIED); + done(); + }); + }); + + it('should not retry retryable INTERNAL errors when retryCodes is explicitly empty', done => { + const attemptStream = through.obj(); + let requestCount = 0; + + const stream = batchWriteStream( + [group0], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: { + retry: { + retryCodes: [], + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + }, + }, + }, + }, + ); + + const eosErr = makeServiceError( + grpc.status.INTERNAL, + 'Received unexpected EOS on DATA frame from server', + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err, eosErr); + done(); + }); + + attemptStream.emit('error', eosErr); + }); + + it('should disable retries on UNAVAILABLE when gaxOptions.retry is null', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: {retry: null}, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const unavailableErr = makeServiceError( + grpc.status.UNAVAILABLE, + 'Service currently unavailable', + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(err, unavailableErr); + assert.strictEqual(finalStatus, grpc.status.UNAVAILABLE); + done(); + }); + + attemptStream.emit('error', unavailableErr); + }); + + it('should disable retries on premature stream end when gaxOptions.retry is null', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: {retry: null}, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + const responses: IBatchWriteResponse[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => responses.push(resp)) + .on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.strictEqual(responses.length, 1); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.strictEqual(err.code, grpc.status.UNAVAILABLE); + assert.match(err.message, /ended prematurely/); + assert.strictEqual(finalStatus, grpc.status.UNAVAILABLE); + done(); + }); + + attemptStream.write({indexes: [0], status: {code: grpc.status.OK}}); + attemptStream.end(); + }); + + it('should cancel pending retry timer and active stream when caller destroys stream early', () => { + const clock = sandbox.useFakeTimers(); + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0, group1], + () => { + requestCount++; + return attemptStream; + }, + { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 50, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 100, + totalTimeoutMillis: 5000, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + attemptStream.write({indexes: [0], status: {code: grpc.status.OK}}); + // Trigger retry backoff timer + attemptStream.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Unavailable'), + ); + + // Caller destroys stream while retry timer is waiting + stream.destroy(); + + clock.tick(200); + assert.strictEqual(requestCount, 1); + assert.strictEqual(finalStatus, grpc.status.CANCELLED); + }); + + it('should emit DeadlineError and finalize with DEADLINE_EXCEEDED when retry delay exceeds total timeout', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + [group0], + () => { + requestCount++; + setImmediate(() => { + attemptStream.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + }); + return attemptStream; + }, + { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1000, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 2000, + totalTimeoutMillis: 50, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.ok(err instanceof DeadlineError); + assert.strictEqual(err.code, grpc.status.DEADLINE_EXCEEDED); + assert.strictEqual(finalStatus, grpc.status.DEADLINE_EXCEEDED); + assert.strictEqual((err as DeadlineError).errors.length, 1); + assert.strictEqual( + (err as DeadlineError).errors[0].code, + grpc.status.UNAVAILABLE, + ); + done(); + }); + }); + + it('should emit DeadlineError and finalize with DEADLINE_EXCEEDED when total timeout has already elapsed prior to failure', done => { + const attemptStream = through.obj(); + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + let currentTime = 1000; + sandbox.stub(Date, 'now').callsFake(() => currentTime); + + const stream = batchWriteStream( + [group0], + () => { + requestCount++; + setImmediate(() => { + // Advance time past the 500ms timeout + currentTime += 600; + attemptStream.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + }); + return attemptStream; + }, + { + gaxOptions: { + timeout: 500, + retry: { + backoffSettings: { + initialRetryDelayMillis: 10, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 50, + totalTimeoutMillis: 500, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + assert.strictEqual(requestCount, 1); + assert.ok(err instanceof DeadlineError); + assert.strictEqual(err.code, grpc.status.DEADLINE_EXCEEDED); + assert.strictEqual(finalStatus, grpc.status.DEADLINE_EXCEEDED); + assert.strictEqual((err as DeadlineError).errors.length, 1); + assert.strictEqual( + (err as DeadlineError).errors[0].code, + grpc.status.UNAVAILABLE, + ); + done(); + }); + }); + + it('should cease retrying and emit final error when maxRetries is exhausted', done => { + const attemptStreams: Transform[] = [ + through.obj(), + through.obj(), + through.obj(), + ]; + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const unavailableErr = makeServiceError( + grpc.status.UNAVAILABLE, + 'Service currently unavailable', + ); + + const stream = batchWriteStream( + [group0], + () => { + const current = attemptStreams[requestCount++]; + setImmediate(() => { + current.emit('error', unavailableErr); + }); + return current; + }, + { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + maxRetries: 2, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + // 1 initial attempt + 2 retries = 3 attempts total + assert.strictEqual(requestCount, 3); + assert.strictEqual(err, unavailableErr); + assert.strictEqual(err.code, grpc.status.UNAVAILABLE); + assert.strictEqual(finalStatus, grpc.status.UNAVAILABLE); + done(); + }); + }); + + it('should respect top-level gaxOptions.maxRetries (taking precedence over backoffSettings.maxRetries)', done => { + const attemptStreams: Transform[] = [through.obj(), through.obj()]; + let requestCount = 0; + let finalStatus: grpc.status | undefined; + + const unavailableErr = makeServiceError( + grpc.status.UNAVAILABLE, + 'Service currently unavailable', + ); + + const stream = batchWriteStream( + [group0], + () => { + const current = attemptStreams[requestCount++]; + setImmediate(() => { + current.emit('error', unavailableErr); + }); + return current; + }, + { + gaxOptions: { + maxRetries: 1, + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + maxRetries: 5, + }, + }, + }, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + + stream.on('error', (err: grpc.ServiceError) => { + // 1 initial attempt + 1 retry = 2 attempts total + assert.strictEqual(requestCount, 2); + assert.strictEqual(err, unavailableErr); + assert.strictEqual(finalStatus, grpc.status.UNAVAILABLE); + done(); + }); + }); + + it('should pause active request stream on downstream backpressure and resume on drain', done => { + const totalGroups = 40; // Exceeds through2.obj readable + writable highWaterMark (16 + 16 = 32) + const groups: IMutationGroup[] = Array.from( + {length: totalGroups}, + (_, i) => ({ + mutations: [{insert: {table: `T${i}`}}], + }), + ); + const attemptStream = through.obj(); + const pauseSpy = sandbox.spy(attemptStream, 'pause'); + const resumeSpy = sandbox.spy(attemptStream, 'resume'); + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream(groups, () => attemptStream, { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }); + + const initialResumeCount = resumeSpy.callCount; + + // Push 40 responses while no consumer is reading `stream` to trigger backpressure + for (let i = 0; i < totalGroups; i++) { + attemptStream.write({indexes: [i], status: {code: grpc.status.OK}}); + } + attemptStream.end(); + + assert.strictEqual( + pauseSpy.called, + true, + 'Expected upstream request stream to be paused on backpressure', + ); + + const receivedIndexes: number[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => { + receivedIndexes.push(...(resp.indexes ?? [])); + }) + .on('error', done) + .on('end', () => { + assert.ok( + resumeSpy.callCount > initialResumeCount, + 'Expected upstream request stream to be resumed on drain', + ); + assert.deepStrictEqual( + receivedIndexes, + Array.from({length: totalGroups}, (_, i) => i), + ); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + + it('should handle downstream backpressure cleanly across stream retries', done => { + const totalGroups = 50; + const groups: IMutationGroup[] = Array.from( + {length: totalGroups}, + (_, i) => ({ + mutations: [{insert: {table: `T${i}`}}], + }), + ); + const stream1 = through.obj(); + const stream2 = through.obj(); + const stream1PauseSpy = sandbox.spy(stream1, 'pause'); + const stream2PauseSpy = sandbox.spy(stream2, 'pause'); + const stream2ResumeSpy = sandbox.spy(stream2, 'resume'); + let attemptIndex = 0; + let finalStatus: grpc.status | undefined; + + const stream = batchWriteStream( + groups, + currentGroups => { + attemptIndex++; + if (attemptIndex === 1) { + setImmediate(() => { + // Emit 40 responses with no downstream reader -> triggers backpressure on stream1 + for (let i = 0; i < 40; i++) { + stream1.write({indexes: [i], status: {code: grpc.status.OK}}); + } + assert.strictEqual(stream1PauseSpy.called, true); + stream1.emit( + 'error', + makeServiceError(grpc.status.UNAVAILABLE, 'Temporary disconnect'), + ); + }); + return stream1; + } + + setImmediate(() => { + const initialStream2Resumes = stream2ResumeSpy.callCount; + // userStream buffer is still full -> writing remaining 10 responses pauses stream2 + for (let i = 0; i < currentGroups.length; i++) { + stream2.write({indexes: [i], status: {code: grpc.status.OK}}); + } + stream2.end(); + assert.strictEqual(stream2PauseSpy.called, true); + + // Start consuming downstream so userStream drains and resumes stream2 + const receivedIndexes: number[] = []; + stream + .on('data', (resp: IBatchWriteResponse) => { + receivedIndexes.push(...(resp.indexes ?? [])); + }) + .on('error', done) + .on('end', () => { + assert.ok(stream2ResumeSpy.callCount > initialStream2Resumes); + assert.deepStrictEqual( + receivedIndexes, + Array.from({length: totalGroups}, (_, i) => i), + ); + assert.strictEqual(finalStatus, grpc.status.OK); + done(); + }); + }); + return stream2; + }, + { + gaxOptions: fastGaxOptions, + onTerminate: status => { + finalStatus = status; + }, + }, + ); + }); + + describe('getNextRetryDelayMs', () => { + it('should extract delay from google.rpc.retryinfo-bin metadata when present', () => { + const RetryInfo = getRetryInfo(); + const encoded = RetryInfo.encode({ + retryDelay: {seconds: 2, nanos: 250000000}, + }).finish(); + const metadata = new grpc.Metadata(); + metadata.set(RETRY_INFO_BIN, Buffer.from(encoded)); + + const err = makeServiceError( + grpc.status.UNAVAILABLE, + 'Unavailable', + metadata, + ); + const delayMs = getNextRetryDelayMs(err, 1); + assert.strictEqual(delayMs, 2250); + }); + + it('should fall back to jittered exponential backoff when RetryInfo is absent', () => { + const err = makeServiceError(grpc.status.UNAVAILABLE, 'Unavailable'); + const delayMs = getNextRetryDelayMs(err, 1, { + initialRetryDelayMillis: 100, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 32000, + }); + assert.ok( + delayMs >= 80 && delayMs <= 120, + `Expected delay in [80, 120], got ${delayMs}`, + ); + }); + + it('should fall back to jittered exponential backoff when RetryInfo metadata is corrupted', () => { + const metadata = new grpc.Metadata(); + metadata.set(RETRY_INFO_BIN, Buffer.from('invalid-non-proto-bytes')); + + const err = makeServiceError( + grpc.status.UNAVAILABLE, + 'Unavailable', + metadata, + ); + const delayMs = getNextRetryDelayMs(err, 1, { + initialRetryDelayMillis: 100, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 32000, + }); + assert.ok( + delayMs >= 80 && delayMs <= 120, + `Expected delay in [80, 120], got ${delayMs}`, + ); + }); + + it('should fall back to jittered exponential backoff when metadata is undefined or lacks a get method', () => { + const errWithoutMetadata = Object.assign(new Error('Unavailable'), { + code: grpc.status.UNAVAILABLE, + }) as grpc.ServiceError; + const errWithPlainObjectMetadata = Object.assign( + new Error('Unavailable'), + { + code: grpc.status.UNAVAILABLE, + metadata: {} as unknown as grpc.Metadata, + }, + ) as grpc.ServiceError; + + for (const err of [errWithoutMetadata, errWithPlainObjectMetadata]) { + const delayMs = getNextRetryDelayMs(err, 1, { + initialRetryDelayMillis: 100, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 32000, + }); + assert.ok( + delayMs >= 80 && delayMs <= 120, + `Expected delay in [80, 120], got ${delayMs}`, + ); + } + }); + }); +}); diff --git a/handwritten/spanner/test/database.ts b/handwritten/spanner/test/database.ts index d5b66fae4bce..b9641e77a6ca 100644 --- a/handwritten/spanner/test/database.ts +++ b/handwritten/spanner/test/database.ts @@ -276,6 +276,7 @@ describe('Database', () => { const SPANNER = { routeToLeaderEnabled: true, options: {}, + _initMetricsTracer: util.noop, } as {} as Spanner; const INSTANCE = { @@ -745,7 +746,12 @@ describe('Database', () => { }); it('should call `requestStream` with correct arguments', () => { - const expectedGaxOpts = extend(true, {}, options?.gaxOptions); + const expectedGaxOpts = extend(true, {}, options?.gaxOptions, { + retryRequestOptions: { + noResponseRetries: 0, + retries: 0, + }, + }); const expectedReqOpts: google.spanner.v1.IBatchWriteRequest = { session: fakeSession!.formattedName_!, mutationGroups: mutationGroups.map(mg => mg.proto()), @@ -789,7 +795,12 @@ describe('Database', () => { assert.strictEqual(args.method, 'batchWrite'); assert.deepStrictEqual(args.reqOpts, expectedReqOpts); assert.strictEqual(args.reqOpts.excludeTxnFromChangeStreams, undefined); - assert.deepStrictEqual(args.gaxOpts, {}); + assert.deepStrictEqual(args.gaxOpts, { + retryRequestOptions: { + noResponseRetries: 0, + retries: 0, + }, + }); }); it('should not include leader-aware routing header when routeToLeaderEnabled is false', () => { @@ -810,23 +821,44 @@ describe('Database', () => { ); }); - it('should return error when passing an empty list of mutationGroups', done => { - const fakeError = new Error('err'); - database.batchWriteAtLeastOnce([], options).on('error', error => { - assert.strictEqual(error, fakeError); - done(); - }); - fakeDataStream.emit('error', fakeError); + it('should immediately end stream with zero RPCs and no session when passing an empty list of mutationGroups', done => { + database + .batchWriteAtLeastOnce([], options) + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(getSessionStub.callCount, 0); + assert.strictEqual(requestStreamStub.callCount, 0); + done(); + }); + }); + + it('should immediately end stream with zero RPCs and no session when mutationGroups is omitted', done => { + database + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .batchWriteAtLeastOnce(undefined as any, options) + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(getSessionStub.callCount, 0); + assert.strictEqual(requestStreamStub.callCount, 0); + done(); + }); }); it('should return data when passing a valid list of mutationGroups', done => { + const fakeResponse = {indexes: [0, 1]}; database .batchWriteAtLeastOnce(mutationGroups, options) .on('data', data => { - assert.strictEqual(data, 'test'); + assert.deepStrictEqual(data, fakeResponse); done(); }); - fakeDataStream.emit('data', 'test'); + fakeDataStream.emit('data', fakeResponse); }); it('should emit correct event based on valid/invalid list of mutationGroups', done => { @@ -863,21 +895,192 @@ describe('Database', () => { SingerId: 3, AlbumTitle: 'Go, Go, Go', }); + const fakeResponse = {indexes: [0]}; database .batchWriteAtLeastOnce( [FakeMutationGroup1, FakeMutationGroup2], options, ) .on('data', data => { - assert.strictEqual(data, 'testData'); + assert.deepStrictEqual(data, fakeResponse); }) .on('error', err => { assert.strictEqual(err, fakeError); }); - fakeDataStream.emit('data', 'testData'); + fakeDataStream.emit('data', fakeResponse); fakeDataStream.emit('error', fakeError); done(); }); + + it('should retry only remaining unacknowledged mutationGroups on UNAVAILABLE and increment attempt in request ID header', done => { + const stream1 = through.obj(); + const stream2 = through.obj(); + requestStreamStub.onCall(0).returns(stream1); + requestStreamStub.onCall(1).callsFake(() => { + setImmediate(() => { + stream2.write({indexes: [0], status: {code: grpc.status.OK}}); + stream2.end(); + }); + return stream2; + }); + + const retryOptions: BatchWriteOptions = { + ...options, + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + }, + }, + }, + }; + + const receivedResponses: google.spanner.v1.IBatchWriteResponse[] = []; + database + .batchWriteAtLeastOnce(mutationGroups, retryOptions) + .on('data', (resp: google.spanner.v1.IBatchWriteResponse) => { + receivedResponses.push(resp); + }) + .on('error', done) + .on('end', () => { + assert.strictEqual(requestStreamStub.callCount, 2); + + const firstCallArgs = requestStreamStub.getCall(0).args[0]; + const secondCallArgs = requestStreamStub.getCall(1).args[0]; + + assert.deepStrictEqual( + firstCallArgs.reqOpts.mutationGroups, + mutationGroups.map(mg => mg.proto()), + ); + assert.strictEqual(firstCallArgs.gaxOpts.retry, undefined); + assert.ok( + typeof firstCallArgs.gaxOpts.timeout === 'number' && + firstCallArgs.gaxOpts.timeout > 0 && + firstCallArgs.gaxOpts.timeout <= 5000, + ); + assert.deepStrictEqual(firstCallArgs.gaxOpts.retryRequestOptions, { + noResponseRetries: 0, + retries: 0, + }); + assert.strictEqual( + firstCallArgs.headers[X_GOOG_SPANNER_REQUEST_ID_HEADER], + craftRequestId(1, 1, 1, 1), + ); + + // Second attempt only sends mutationGroup2 (unacknowledged) + assert.deepStrictEqual(secondCallArgs.reqOpts.mutationGroups, [ + mutationGroup2.proto(), + ]); + assert.ok( + typeof secondCallArgs.gaxOpts.timeout === 'number' && + secondCallArgs.gaxOpts.timeout > 0 && + secondCallArgs.gaxOpts.timeout <= firstCallArgs.gaxOpts.timeout, + ); + // Same nthRequest (1), attempt incremented to 2 + assert.strictEqual( + secondCallArgs.headers[X_GOOG_SPANNER_REQUEST_ID_HEADER], + craftRequestId(1, 1, 1, 2), + ); + + assert.strictEqual(receivedResponses.length, 2); + assert.deepStrictEqual(receivedResponses[0].indexes, [0]); + // Retry-local index 0 in attempt 2 is translated back to original index 1 + assert.deepStrictEqual(receivedResponses[1].indexes, [1]); + done(); + }); + + stream1.write({indexes: [0], status: {code: grpc.status.OK}}); + stream1.emit( + 'error', + Object.assign(new Error('Unavailable'), { + code: grpc.status.UNAVAILABLE, + }), + ); + }); + + it('should initialize and pass metricsTracer across all retry attempts in batchWriteAtLeastOnce', done => { + const stream1 = through.obj(); + const stream2 = through.obj(); + const fakeTracer = { + recordOperationCompletion: sandbox.spy(), + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const spanner = (database as any)._getSpanner(); + sandbox.stub(spanner, '_initMetricsTracer').returns(fakeTracer); + + requestStreamStub.onCall(0).returns(stream1); + requestStreamStub.onCall(1).callsFake(() => { + setImmediate(() => { + stream2.write({indexes: [0], status: {code: grpc.status.OK}}); + stream2.end(); + }); + return stream2; + }); + + const retryOptions: BatchWriteOptions = { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 5, + totalTimeoutMillis: 5000, + }, + }, + }, + }; + + database + .batchWriteAtLeastOnce(mutationGroups, retryOptions) + .on('data', () => {}) + .on('error', done) + .on('end', () => { + assert.strictEqual(requestStreamStub.callCount, 2); + const firstCallArgs = requestStreamStub.getCall(0).args[0]; + const secondCallArgs = requestStreamStub.getCall(1).args[0]; + + assert.strictEqual(firstCallArgs.metricsTracer, fakeTracer); + assert.strictEqual(secondCallArgs.metricsTracer, fakeTracer); + assert.strictEqual( + fakeTracer.recordOperationCompletion.calledOnce, + true, + ); + assert.strictEqual( + fakeTracer.recordOperationCompletion.firstCall.args[0], + grpc.status.OK, + ); + done(); + }); + + stream1.write({indexes: [0], status: {code: grpc.status.OK}}); + stream1.emit( + 'error', + Object.assign(new Error('Unavailable'), { + code: grpc.status.UNAVAILABLE, + }), + ); + }); + + it('should not call `requestStream` or initialize metrics if stream is destroyed before `getSession` resolves', done => { + let sessionCallback: + ((err: Error | null, session?: FakeSession) => void) | null = null; + getSessionStub.callsFake(callback => { + sessionCallback = callback; + }); + + const stream = database.batchWriteAtLeastOnce(mutationGroups, options); + stream.destroy(); + + setImmediate(() => { + assert.ok(sessionCallback); + sessionCallback!(null, fakeSession); + assert.strictEqual(requestStreamStub.callCount, 0); + done(); + }); + }); }); describe('writeAtLeastOnce', () => { diff --git a/handwritten/spanner/test/metrics/metrics-tracer.ts b/handwritten/spanner/test/metrics/metrics-tracer.ts index 9037e3ecdd0a..429effc320e4 100644 --- a/handwritten/spanner/test/metrics/metrics-tracer.ts +++ b/handwritten/spanner/test/metrics/metrics-tracer.ts @@ -281,6 +281,44 @@ describe('MetricsTracer', () => { ); }); + it('should override operation status when finalStatus is passed to recordOperationCompletion', () => { + tracer.recordOperationStart(); + tracer.recordAttemptStart(); + tracer.recordAttemptCompletion(Status.UNAVAILABLE); + tracer.recordOperationCompletion(Status.DEADLINE_EXCEEDED); + + assert.strictEqual(fakeAttemptCounter.add.calledOnce, true); + assert.strictEqual(fakeOperationCounter.add.calledOnce, true); + const [[, operationCounterAttributes]] = fakeOperationCounter.add.args; + assert.strictEqual( + operationCounterAttributes[Constants.METRIC_LABEL_KEY_STATUS], + 'DEADLINE_EXCEEDED', + ); + }); + + it('should finalize an in-flight uncompleted attempt when finalStatus is passed to recordOperationCompletion', () => { + tracer.recordOperationStart(); + tracer.recordAttemptStart(); + // Stream cancelled before recordAttemptCompletion was called by interceptor + tracer.recordOperationCompletion(Status.CANCELLED); + + assert.strictEqual(fakeAttemptCounter.add.calledOnce, true); + assert.strictEqual(fakeAttemptLatency.record.calledOnce, true); + assert.strictEqual(fakeOperationCounter.add.calledOnce, true); + assert.strictEqual(fakeOperationLatency.record.calledOnce, true); + + const [[, attemptAttrs]] = fakeAttemptCounter.add.args; + const [[, operationAttrs]] = fakeOperationCounter.add.args; + assert.strictEqual( + attemptAttrs[Constants.METRIC_LABEL_KEY_STATUS], + 'CANCELLED', + ); + assert.strictEqual( + operationAttrs[Constants.METRIC_LABEL_KEY_STATUS], + 'CANCELLED', + ); + }); + it('should safely do nothing if recordOperationCompletion is called without active operation', () => { tracer.currentOperation = null; assert.doesNotThrow(() => { diff --git a/handwritten/spanner/test/mockserver/mockspanner.ts b/handwritten/spanner/test/mockserver/mockspanner.ts index 4f6f64378d2f..d245800ded1e 100644 --- a/handwritten/spanner/test/mockserver/mockspanner.ts +++ b/handwritten/spanner/test/mockserver/mockspanner.ts @@ -31,7 +31,7 @@ import Any = google.protobuf.Any; import QueryMode = google.spanner.v1.ExecuteSqlRequest.QueryMode; import NullValue = google.protobuf.NullValue; import {ExecuteSqlRequest, ReadRequest} from '../../src/transaction'; -import {getProtoPath, getRetryInfo} from '../../src/protos'; +import {getProtoPath, getRetryInfo, RETRY_INFO_BIN} from '../../src/protos'; const RetryInfo = getRetryInfo(); @@ -57,7 +57,6 @@ const packageDefinition = protoLoader.loadSync(PROTO_PATH, { }); const protoDescriptor = grpc.loadPackageDefinition(packageDefinition); const spannerProtoDescriptor = protoDescriptor['google']['spanner']['v1']; -const RETRY_INFO_BIN = 'google.rpc.retryinfo-bin'; const RETRY_INFO_TYPE = 'type.googleapis.com/google.rpc.retryinfo'; /** @@ -577,6 +576,7 @@ export class MockSpanner { null, protobuf.BatchCreateSessionsResponse.create({session: sessions}), ); + return; }) .catch(err => { callback(err); @@ -597,6 +597,7 @@ export class MockSpanner { call.request!.session?.multiplexed ?? false, ), ); + return; }) .catch(err => { callback(err); @@ -616,6 +617,7 @@ export class MockSpanner { } else { callback(MockSpanner.createSessionNotFoundError(call.request!.name)); } + return; }) .catch(err => { callback(err); @@ -640,6 +642,7 @@ export class MockSpanner { }), }), ); + return; }) .catch(err => { callback(err); @@ -801,6 +804,7 @@ export class MockSpanner { ); } call.end(); + return; }) .catch(err => { call.sendMetadata(new Metadata()); @@ -992,6 +996,7 @@ export class MockSpanner { status: statementStatus, }), ); + return; }) .catch(err => { callback(err); @@ -1123,6 +1128,7 @@ export class MockSpanner { ); } call.end(); + return; }) .catch(err => { call.sendMetadata(new Metadata()); @@ -1151,6 +1157,7 @@ export class MockSpanner { } else { callback(null, res); } + return; }) .catch(err => { callback(err); @@ -1212,6 +1219,7 @@ export class MockSpanner { MockSpanner.createSessionNotFoundError(call.request!.session), ); } + return; }) .catch(err => { callback(err); @@ -1256,6 +1264,7 @@ export class MockSpanner { partitions: [{partitionToken: Buffer.from('mock-token')}], }); callback(null, response); + return; }) .catch(err => callback(err)); } @@ -1271,6 +1280,7 @@ export class MockSpanner { partitions: [{partitionToken: Buffer.from('mock-token')}], }); callback(null, response); + return; }) .catch(err => callback(err)); } @@ -1284,13 +1294,37 @@ export class MockSpanner { this.pushRequest(call.request, call.metadata); this.simulateExecutionTime(this.batchWrite.name) .then(() => { - const response = protobuf.BatchWriteResponse.create({ - commitTimestamp: now(), - }); - call.write(response); + const groupCount = call.request?.mutationGroups?.length ?? 0; + let metadataSent = false; + const ensureMetadataSent = () => { + if (!metadataSent) { + call.sendMetadata(new Metadata()); + metadataSent = true; + } + }; + for (let index = 0; index < groupCount; index++) { + const streamErr = this.shiftStreamError(this.batchWrite.name, index); + if (streamErr) { + ensureMetadataSent(); + call.emit('error', streamErr); + break; + } + ensureMetadataSent(); + const response = protobuf.BatchWriteResponse.create({ + indexes: [index], + status: Status.create({code: grpc.status.OK}), + commitTimestamp: now(), + }); + call.write(response); + } call.end(); + return; }) - .catch(err => call.destroy(err)); + .catch(err => { + call.sendMetadata(new Metadata()); + call.emit('error', err); + call.end(); + }); } private _updateTransaction( diff --git a/handwritten/spanner/test/request-stream-coordinator.ts b/handwritten/spanner/test/request-stream-coordinator.ts index 69c8de388780..27d05afb8eda 100644 --- a/handwritten/spanner/test/request-stream-coordinator.ts +++ b/handwritten/spanner/test/request-stream-coordinator.ts @@ -177,6 +177,107 @@ describe('RequestStreamCoordinator', () => { ); }); + it('should not call recordOperationCompletion when isExternalTracer is true', () => { + const stream = streamEvents(through.obj()); + const tracer = createMockTracer(); + const coordinator = new RequestStreamCoordinator(stream, tracer, true); + coordinator.setup(); + + stream.destroy(); + coordinator.completeOperation(); + + assert.strictEqual( + (tracer.recordOperationCompletion as sinon.SinonSpy).called, + false, + ); + }); + + it('should call _initMetricsTracer when config.metricsTracer is undefined', () => { + const spanner = new Spanner({projectId: 'test-project'}); + const fakeTracer = createMockTracer(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (spanner as any)._initMetricsTracer = sandbox.stub().returns(fakeTracer); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const config: any = {client: 'SpannerClient', method: 'batchWrite'}; + spanner.requestStream(config); + + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((spanner as any)._initMetricsTracer as sinon.SinonStub).calledOnce, + true, + ); + }); + + it('should reuse existing config.metricsTracer and not call _initMetricsTracer when config.metricsTracer is provided', () => { + const spanner = new Spanner({projectId: 'test-project'}); + const existingTracer = createMockTracer(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (spanner as any)._initMetricsTracer = sandbox.stub(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const config: any = { + client: 'SpannerClient', + method: 'batchWrite', + metricsTracer: existingTracer, + }; + spanner.requestStream(config); + + assert.strictEqual(config.metricsTracer, existingTracer); + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((spanner as any)._initMetricsTracer as sinon.SinonStub).called, + false, + ); + }); + + it('should not call _initMetricsTracer when metricsTracer key is explicitly set to undefined', () => { + const spanner = new Spanner({projectId: 'test-project'}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (spanner as any)._initMetricsTracer = sandbox.stub(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const config: any = { + client: 'SpannerClient', + method: 'batchWrite', + metricsTracer: undefined, + }; + spanner.requestStream(config); + + assert.strictEqual(config.metricsTracer, undefined); + assert.strictEqual( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((spanner as any)._initMetricsTracer as sinon.SinonStub).called, + false, + ); + }); + + it('should not throw if callStream is a StreamProxy wrapping a PassThrough without cancel()', () => { + const stream = streamEvents(through.obj()); + const tracer = createMockTracer(); + const coordinator = new RequestStreamCoordinator(stream, tracer); + coordinator.setup(); + + const emitter = new EventEmitter(); + const streamProxyWithoutInnerCancel = Object.assign(emitter, { + stream: through.obj(), // PassThrough/Transform without .cancel() + cancel: function () { + // Simulates google-gax StreamProxy.cancel() calling this.stream.cancel() + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this as any).stream.cancel(); + }, + destroy: sandbox.spy(), + pipe: sandbox.spy((destination: unknown) => destination), + destroyed: false, + }) as unknown as CancellableStream; + + coordinator.attachCallStream(streamProxyWithoutInnerCancel); + + assert.doesNotThrow(() => { + stream.destroy(); + }); + }); + it('should invoke requestFn and attach callStream if stream is active', () => { const stream = streamEvents(through.obj()); const tracer = createMockTracer(); @@ -846,4 +947,132 @@ describe('RequestStreamCoordinator with in-memory mock Spanner server', () => { ); assert.strictEqual(operationCancelled?.value, 1); }); + + it('should record 1 operation count and N attempt counts when batchWriteAtLeastOnce retries on UNAVAILABLE', async () => { + const {MutationGroup} = require('../src/transaction'); + const mg0 = new MutationGroup(); + mg0.upsert('FOO', {Id: '1', Name: 'One'}); + const mg1 = new MutationGroup(); + mg1.upsert('FOO', {Id: '2', Name: 'Two'}); + + const unavailableErr = Object.assign(new Error('Mid-stream unavailable'), { + code: grpc.status.UNAVAILABLE, + streamIndex: 1, + metadata: mock.MockSpanner.createMinimalRetryDelayMetadata(), + }) as mock.MockError; + + spannerMock.setExecutionTime( + spannerMock.batchWrite, + mock.SimulatedExecutionTime.ofError(unavailableErr), + ); + + const stream = database.batchWriteAtLeastOnce([mg0, mg1]); + const responses: unknown[] = []; + + await new Promise((resolve, reject) => { + stream.on('data', row => responses.push(row)); + stream.on('end', () => resolve()); + stream.on('error', reject); + }); + + assert.strictEqual(responses.length, 2); + + const resourceMetrics = await waitForMetrics(reader, resMetrics => { + const opPoints = getMetricDataPoints( + resMetrics, + METRIC_NAME_OPERATION_COUNT, + ); + return opPoints.some( + dp => + dp.attributes.method === 'batchWrite' && + dp.attributes.status === 'OK', + ); + }); + + const attemptPoints = getMetricDataPoints( + resourceMetrics, + METRIC_NAME_ATTEMPT_COUNT, + ); + const operationPoints = getMetricDataPoints( + resourceMetrics, + METRIC_NAME_OPERATION_COUNT, + ); + + const attemptUnavailable = findDataPoint( + attemptPoints, + 'batchWrite', + 'UNAVAILABLE', + ); + const attemptOk = findDataPoint(attemptPoints, 'batchWrite', 'OK'); + const operationOk = findDataPoint(operationPoints, 'batchWrite', 'OK'); + const operationUnavailable = findDataPoint( + operationPoints, + 'batchWrite', + 'UNAVAILABLE', + ); + + assert.strictEqual(attemptUnavailable?.value, 1); + assert.strictEqual(attemptOk?.value, 1); + assert.strictEqual(operationOk?.value, 1); + assert.strictEqual(operationUnavailable, undefined); + }); + + it('should record CANCELLED for both attempt and operation when batchWriteAtLeastOnce stream is cancelled early', async () => { + const {MutationGroup} = require('../src/transaction'); + const mg0 = new MutationGroup(); + mg0.upsert('FOO', {Id: '1', Name: 'One'}); + const mg1 = new MutationGroup(); + mg1.upsert('FOO', {Id: '2', Name: 'Two'}); + + const stream = database.batchWriteAtLeastOnce([mg0, mg1]); + + // Read first response and break out of loop (destroying the stream early) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _ of stream) { + break; + } + + const resourceMetrics = await waitForMetrics(reader, resMetrics => { + const opPoints = getMetricDataPoints( + resMetrics, + METRIC_NAME_OPERATION_COUNT, + ); + return opPoints.some( + dp => + dp.attributes.method === 'batchWrite' && + dp.attributes.status === 'CANCELLED', + ); + }); + + const attemptPoints = getMetricDataPoints( + resourceMetrics, + METRIC_NAME_ATTEMPT_COUNT, + ); + const operationPoints = getMetricDataPoints( + resourceMetrics, + METRIC_NAME_OPERATION_COUNT, + ); + + const attemptCancelled = findDataPoint( + attemptPoints, + 'batchWrite', + 'CANCELLED', + ); + const operationCancelled = findDataPoint( + operationPoints, + 'batchWrite', + 'CANCELLED', + ); + + assert.ok( + attemptCancelled, + 'Expected batchWrite attempt with status CANCELLED on early stream destruction', + ); + assert.strictEqual(attemptCancelled?.value, 1); + assert.ok( + operationCancelled, + 'Expected batchWrite operation with status CANCELLED on early stream destruction', + ); + assert.strictEqual(operationCancelled?.value, 1); + }); }); diff --git a/handwritten/spanner/test/spanner.ts b/handwritten/spanner/test/spanner.ts index 8863a78b3eee..4dedee9246d1 100644 --- a/handwritten/spanner/test/spanner.ts +++ b/handwritten/spanner/test/spanner.ts @@ -2278,11 +2278,115 @@ describe('Spanner with mock server', () => { .on('data', response => { // ensure that response is coming assert.notEqual(response.commitTimestamp, null); + assert.deepStrictEqual(response.indexes, [0]); // multiplexed session will get created by default assert.notEqual(multiplexedSession._multiplexedSession, null); }) .on('end', done); }); + + it('should retry mid-stream UNAVAILABLE and resume with only unacknowledged mutation groups', done => { + const mg0 = new MutationGroup(); + mg0.upsert('FOO', {Id: '1', Name: 'One'}); + const mg1 = new MutationGroup(); + mg1.upsert('FOO', {Id: '2', Name: 'Two'}); + const mg2 = new MutationGroup(); + mg2.upsert('FOO', {Id: '3', Name: 'Three'}); + + const unavailableErr = { + message: 'Stream broken mid-flight', + code: grpc.status.UNAVAILABLE, + streamIndex: 1, + metadata: mock.MockSpanner.createMinimalRetryDelayMetadata(), + } as mock.MockError; + + spannerMock.setExecutionTime( + spannerMock.batchWrite, + mock.SimulatedExecutionTime.ofError(unavailableErr), + ); + + const database = newTestDatabase({min: 1, max: 1}); + const responses: protos.google.spanner.v1.IBatchWriteResponse[] = []; + + database + .batchWriteAtLeastOnce([mg0, mg1, mg2], { + gaxOptions: { + retry: { + backoffSettings: { + initialRetryDelayMillis: 1, + retryDelayMultiplier: 1.3, + maxRetryDelayMillis: 10, + maxRetries: 3, + }, + }, + }, + }) + .on('error', done) + .on( + 'data', + (response: protos.google.spanner.v1.IBatchWriteResponse) => { + responses.push(response); + }, + ) + .on('end', () => { + assert.strictEqual(responses.length, 3); + assert.deepStrictEqual(responses[0].indexes, [0]); + assert.deepStrictEqual(responses[1].indexes, [1]); + assert.deepStrictEqual(responses[2].indexes, [2]); + + const batchWriteRequests = spannerMock + .getRequests() + .filter( + req => + (req as protos.google.spanner.v1.BatchWriteRequest) + .mutationGroups !== undefined, + ) as protos.google.spanner.v1.BatchWriteRequest[]; + + assert.strictEqual(batchWriteRequests.length, 2); + assert.strictEqual(batchWriteRequests[0].mutationGroups.length, 3); + assert.strictEqual(batchWriteRequests[1].mutationGroups.length, 2); + done(); + }); + }); + + it('should not retry immediate non-retryable error in google-gax retry-request', done => { + const mg0 = new MutationGroup(); + mg0.upsert('FOO', {Id: '1', Name: 'One'}); + + const invalidArgErr = { + message: 'Invalid mutation group', + code: grpc.status.INVALID_ARGUMENT, + } as mock.MockError; + + spannerMock.setExecutionTime( + spannerMock.batchWrite, + mock.SimulatedExecutionTime.ofErrors([ + invalidArgErr, + invalidArgErr, + invalidArgErr, + ]), + ); + + const database = newTestDatabase({min: 1, max: 1}); + + database + .batchWriteAtLeastOnce([mg0]) + .on('data', () => { + done(new Error('Should not emit data')); + }) + .on('error', (err: ServiceError) => { + assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT); + const batchWriteRequests = spannerMock + .getRequests() + .filter( + req => + (req as protos.google.spanner.v1.BatchWriteRequest) + .mutationGroups !== undefined, + ) as protos.google.spanner.v1.BatchWriteRequest[]; + assert.strictEqual(batchWriteRequests.length, 1); + done(); + }); + }); }); });