diff --git a/core/packages/gax/src/fallbackRest.ts b/core/packages/gax/src/fallbackRest.ts index 7429924e35df..a1d5edb58efa 100644 --- a/core/packages/gax/src/fallbackRest.ts +++ b/core/packages/gax/src/fallbackRest.ts @@ -19,7 +19,7 @@ import * as serializer from 'proto3-json-serializer'; import {defaultToObjectOptions} from './fallback'; import {FetchParameters, FetchParametersMethod} from './fallbackServiceStub'; -import {GoogleError} from './googleError'; +import {GoogleError, setRawHttpResponsePayload} from './googleError'; import {rpcCodeFromHttpStatusCode} from './status'; import {getMatchedUrlTemplate, transcode} from './transcoding'; @@ -160,7 +160,7 @@ export function decodeResponse( if (ok) { throw err; } - throw _undecodableErrorResponse( + const undecodableError = _undecodableErrorResponse( rpc, httpStatusCode, `the response body is not JSON: ${ @@ -170,6 +170,8 @@ export function decodeResponse( }`, err, ); + setRawHttpResponsePayload(undecodableError, decodedString); + throw undecodableError; } if (!ok) { let error: GoogleError; @@ -179,12 +181,14 @@ export function decodeResponse( // `parseHttpError` indexes into the body, so JSON that parses but is not // an object — `null`, a string, a number — fails there for the same // reason an HTML body fails above, and must not lose the status either. - throw _undecodableErrorResponse( + const undecodableError = _undecodableErrorResponse( rpc, httpStatusCode, 'the response body is not a JSON error object', err, ); + setRawHttpResponsePayload(undecodableError, decodedString); + throw undecodableError; } // `parseHttpError` reads the status out of the response body and maps it // onto the gRPC `code`, keeping no record of the status the transport @@ -197,6 +201,7 @@ export function decodeResponse( if (httpStatusCode !== undefined) { error.httpStatusCode = httpStatusCode; } + setRawHttpResponsePayload(error, decodedString); throw error; } const message = serializer.fromProto3JSON(rpc.resolvedResponseType!, json); diff --git a/core/packages/gax/src/fallbackServiceStub.ts b/core/packages/gax/src/fallbackServiceStub.ts index cf32e5dad8cd..a4180a2ff6a0 100644 --- a/core/packages/gax/src/fallbackServiceStub.ts +++ b/core/packages/gax/src/fallbackServiceStub.ts @@ -24,7 +24,7 @@ import * as serializer from 'proto3-json-serializer'; import {isNodeJS} from './featureDetection'; import {StreamArrayParser} from './streamArrayParser'; import {defaultToObjectOptions} from './fallback'; -import {GoogleError} from './googleError'; +import {GoogleError, setRawHttpResponsePayload} from './googleError'; import {setAttemptHttpMethod} from './observability/TracerHelper'; import {rpcCodeFromHttpStatusCode, Status} from './status'; import {pipeline, PipelineSource} from 'stream'; @@ -234,6 +234,43 @@ function _parseHttpErrorFromRejection( return undefined; } +const GAXIOS_REDACTED_BODY = + '< - See `errorRedactor` option in `gaxios` for configuration>.'; + +function _isStreamLike(value: unknown): boolean { + if (!value || typeof value !== 'object') { + return false; + } + const stream = value as {pipe?: unknown; getReader?: unknown}; + return ( + typeof stream.pipe === 'function' || typeof stream.getReader === 'function' + ); +} + +function _extractRawPayloadFromRejection( + err: Error, + responseData: unknown, + httpStatus: number, +): unknown { + if ( + responseData !== undefined && + responseData !== null && + responseData !== '' && + responseData !== GAXIOS_REDACTED_BODY && + !_isStreamLike(responseData) + ) { + return responseData; + } + if ( + err.message.length > 0 && + err.message !== GAXIOS_REDACTED_BODY && + err.message !== `Request failed with status code ${httpStatus}` + ) { + return err.message; + } + return undefined; +} + function _toGoogleError(err: unknown, outcome: CallOutcome): unknown { if (err instanceof GoogleError) { return err; @@ -264,13 +301,22 @@ function _toGoogleError(err: unknown, outcome: CallOutcome): unknown { ? fetchError.status : fetchError.response?.status; if (typeof httpStatus === 'number') { + const responseData = fetchError.response?.data; const parsedError = - _parseHttpErrorFromRejection(err, fetchError.response?.data) ?? error; + _parseHttpErrorFromRejection(err, responseData) ?? error; parsedError.cause = err; if (typeof parsedError.code !== 'number') { parsedError.code = rpcCodeFromHttpStatusCode(httpStatus); } parsedError.httpStatusCode = httpStatus; + const rawPayload = _extractRawPayloadFromRejection( + err, + responseData, + httpStatus, + ); + if (rawPayload !== undefined) { + setRawHttpResponsePayload(parsedError, rawPayload); + } return parsedError; } diff --git a/core/packages/gax/src/googleError.ts b/core/packages/gax/src/googleError.ts index 961fab6bdf75..58e6a11507d8 100644 --- a/core/packages/gax/src/googleError.ts +++ b/core/packages/gax/src/googleError.ts @@ -29,6 +29,42 @@ const UNKNOWN_TYPE_ENCONDED_ERROR_PREFIX = 'Unknown type encoded in'; const UNKNOWN_TYPE_NO_SUCH_TYPE = 'no such type'; const NUM_OF_PARTS_IN_PROTO_TYPE_NAME = 2; +const RAW_HTTP_RESPONSE_PAYLOAD = Symbol.for( + 'google-gax.rawHttpResponsePayload', +); + +/** + * Attaches the full raw HTTP response body non-enumerably to a GoogleError so + * telemetry debug logging can report `response.payload` without altering + * standard error formatting or serialization. + */ +export function setRawHttpResponsePayload( + err: GoogleError, + payload: unknown, +): void { + try { + Object.defineProperty(err, RAW_HTTP_RESPONSE_PAYLOAD, { + value: payload, + configurable: true, + enumerable: false, + writable: true, + }); + } catch { + // Fail open if err is frozen, non-extensible, or traps defineProperty. + } +} + +/** + * Reads the raw HTTP response body previously attached by + * {@link setRawHttpResponsePayload}, if any. + */ +export function getRawHttpResponsePayload(err: object): unknown { + try { + return (err as Record)[RAW_HTTP_RESPONSE_PAYLOAD]; + } catch { + return undefined; + } +} export class GoogleError extends Error { code?: Status; diff --git a/core/packages/gax/src/observability/LoggerHelper.ts b/core/packages/gax/src/observability/LoggerHelper.ts index 1c15efafff07..c1c09ce3703a 100644 --- a/core/packages/gax/src/observability/LoggerHelper.ts +++ b/core/packages/gax/src/observability/LoggerHelper.ts @@ -31,6 +31,7 @@ import { isPreConnectionFailure, resolveErrorInfoDomain, resolveErrorInfoMetadataAttributes, + resolveErrorResponsePayload, resolveErrorType, resolveExceptionMessage, resolveExceptionStacktrace, @@ -179,6 +180,18 @@ function emitGaxLog( } } +/** + * Resolves a `LogSeverity` enum value. Until a `google-logging-utils` release + * containing #9718 is published to npm (this repo uses `link-workspace-packages=false`), + * the published package exports `LogSeverity` as a type only while each enum member's + * runtime value equals its uppercase level string. + */ +function resolveLogSeverity( + level: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR', +): loggingUtils.LogSeverity { + return level as loggingUtils.LogSeverity; +} + /** * Returns the `google-logging-utils` logger for the given GCP client service, * defaulting to the `'google-gax'` namespace when no service name is provided. @@ -216,12 +229,15 @@ export function getGaxLogger( * @param {unknown} err - The error thrown or reported for the call. * @param {AttemptTraceContext} dynamicArgs - Dynamic context for the RPC call or attempt. * @param {StaticTraceContext} [staticArgs={}] - Static context for the client library. + * @param {boolean} [includeExceptionAttributes=true] - Whether to populate `exception.*` + * attributes (set on L3 client call logs, omitted on L4 transport attempt logs). * @returns {LogPayload} The populated log payload attributes with undefined fields omitted. */ export function buildLogAttributes( err: unknown, dynamicArgs: AttemptTraceContext, staticArgs: StaticTraceContext = {}, + includeExceptionAttributes = true, ): LogPayload { const {rawAddress, urlDomain} = resolveEndpointAndDomain( dynamicArgs, @@ -275,14 +291,16 @@ export function buildLogAttributes( } Object.assign(attributes, resolveErrorInfoMetadataAttributes(err)); - attributes['exception.type'] = resolveExceptionType(err); - attributes['exception.message'] = resolveExceptionMessage( - err, - dynamicArgs.rpcType, - ); - const stacktrace = resolveExceptionStacktrace(err); - if (stacktrace !== undefined) { - attributes['exception.stacktrace'] = stacktrace; + if (includeExceptionAttributes) { + attributes['exception.type'] = resolveExceptionType(err); + attributes['exception.message'] = resolveExceptionMessage( + err, + dynamicArgs.rpcType, + ); + const stacktrace = resolveExceptionStacktrace(err); + if (stacktrace !== undefined) { + attributes['exception.stacktrace'] = stacktrace; + } } return attributes; @@ -391,17 +409,10 @@ export function logCall( // Build structured log attributes with exception details. const attributes = buildLogAttributes(err, dynamicArgs, staticArgs); - const warningSeverity = - ( - loggingUtils as { - LogSeverity?: {WARNING: loggingUtils.LogSeverity}; - } - ).LogSeverity?.WARNING ?? ('WARNING' as loggingUtils.LogSeverity); - // Emit structured WARNING log. logger( { - severity: warningSeverity, + severity: resolveLogSeverity('WARNING'), ...(traceId ? {telemetryTraceId: traceId} : {}), ...(spanId ? {telemetrySpanId: spanId} : {}), other: attributes, @@ -413,3 +424,77 @@ export function logCall( return executeLoggedOperation(fn, emitL3WarnLog, isStreamCall, userCallback); } + +/** + * Executes an individual RPC transport attempt and emits a `DEBUG` log entry + * if the attempt fails across promise, callback, stream, or synchronous execution boundaries. + * + * @template T + * @param {AttemptTraceContext} attemptDynamicArgs - Dynamic context for the RPC attempt. + * @param {StaticTraceContext} staticArgs - Static context for the client library. + * @param {function} fn - The transport attempt operation to execute. Receives the wrapped + * callback when `userCallback` is supplied, otherwise `undefined`. + * @param {boolean} [isStreamCall=false] - Whether the operation returns a stream (`EventEmitter`). + * @param {APICallback} [userCallback] - Optional attempt callback. + * @returns {T} The return value of `fn`. + */ +export function logNetworkAttempt( + attemptDynamicArgs: AttemptTraceContext, + staticArgs: StaticTraceContext, + fn: (loggedCallback?: APICallback) => T, + isStreamCall = false, + userCallback?: APICallback, +): T { + // Snapshot attempt context in case async rejection loses active context. + let initialContext: ReturnType | undefined; + try { + initialContext = context.active(); + } catch { + // Fail open if active context retrieval throws. + } + + const emitL4DebugLog = (err: unknown) => { + // Resolve logger and active T4 attempt span (or outer OpenTelemetry span when GAX tracing is disabled). + const logger = getGaxLogger(staticArgs?.gcpClientService); + const activeTrace = getActiveTraceContext(); + const initialTrace = initialContext + ? getActiveTraceContext(initialContext) + : undefined; + const attemptSpan = + activeTrace?.attemptSpan ?? + initialTrace?.attemptSpan ?? + activeTrace?.span ?? + initialTrace?.span; + const spanContext = attemptSpan?.spanContext(); + const traceId = spanContext?.traceId; + const spanId = spanContext?.spanId; + + // Build structured log attributes without exception details and attach raw attempt response payload when present. + const attributes = buildLogAttributes( + err, + attemptDynamicArgs, + staticArgs, + false, + ); + const responsePayload = resolveErrorResponsePayload(err); + if (responsePayload !== undefined) { + attributes['response.payload'] = responsePayload; + } + + // Emit structured DEBUG log for the failed attempt. + const message = resolveExceptionMessage(err, attemptDynamicArgs.rpcType); + logger( + { + severity: resolveLogSeverity('DEBUG'), + ...(traceId ? {telemetryTraceId: traceId} : {}), + ...(spanId ? {telemetrySpanId: spanId} : {}), + other: attributes, + }, + 'RPC attempt failed: %s', + message || err, + ); + }; + + // Intercept and dispatch execution boundaries for the attempt. + return executeLoggedOperation(fn, emitL4DebugLog, isStreamCall, userCallback); +} diff --git a/core/packages/gax/src/observability/errorUtil.ts b/core/packages/gax/src/observability/errorUtil.ts index e8ccf2d5f8bc..dd60009d5320 100644 --- a/core/packages/gax/src/observability/errorUtil.ts +++ b/core/packages/gax/src/observability/errorUtil.ts @@ -16,7 +16,7 @@ import {Attributes, Span} from '@opentelemetry/api'; import * as protobuf from 'protobufjs'; -import {GoogleError} from '../googleError'; +import {getRawHttpResponsePayload, GoogleError} from '../googleError'; import {Status} from '../status'; /** * System and network error codes mapped to CLIENT_CONNECTION_ERROR. @@ -938,6 +938,106 @@ export function resolveResponsePayload( return bytes ? encodeBase64(bytes) : safeJsonStringify(payload); } +const GAXIOS_REDACTED_BODY = + '< - See `errorRedactor` option in `gaxios` for configuration>.'; + +function isStreamLike(value: unknown): boolean { + if (!value || typeof value !== 'object') { + return false; + } + const stream = value as {pipe?: unknown; getReader?: unknown}; + return ( + typeof stream.pipe === 'function' || typeof stream.getReader === 'function' + ); +} + +/** + * Resolves the `response.payload` attribute from a failed RPC attempt error: + * - For HTTP REST errors translated by GAX fallback, extracts the full raw HTTP + * response payload attached to the {@link GoogleError} (or a direct `response.data` + * payload when usable). + * - For gRPC errors, unwraps `Metadata#getMap()` so header values are serialized + * as scalar strings / base64 rather than single-element arrays from `Metadata#toJSON()`, + * and omits empty `Metadata` instances (such as the empty `new grpc.Metadata()` + * attached by `@grpc/grpc-js` on pre-connection failures like `ECONNREFUSED`). + * + * @param {unknown} e - The error thrown or returned by a failed attempt. + * @returns {string | undefined} The serialized response payload, or undefined if none arrived. + */ +export function resolveErrorResponsePayload(e: unknown): string | undefined { + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object' || seen.has(current)) { + break; + } + seen.add(current); + + const rawHttpPayload = getRawHttpResponsePayload(current); + if (rawHttpPayload !== undefined) { + return resolveResponsePayload(rawHttpPayload); + } + + const errObj = current as { + response?: {data?: unknown} | null; + metadata?: unknown; + cause?: unknown; + }; + + const responseData = + errObj.response && typeof errObj.response === 'object' + ? errObj.response.data + : undefined; + if ( + responseData !== undefined && + responseData !== null && + responseData !== '' && + responseData !== GAXIOS_REDACTED_BODY && + !isStreamLike(responseData) + ) { + return resolveResponsePayload(responseData); + } + + if (errObj.metadata !== undefined && errObj.metadata !== null) { + if (typeof errObj.metadata === 'string') { + if (errObj.metadata.length > 0) { + return errObj.metadata; + } + } else { + const bytes = toBytes(errObj.metadata); + if (bytes) { + if (bytes.byteLength > 0) { + return encodeBase64(bytes); + } + } else if (typeof errObj.metadata === 'object') { + let map: unknown = errObj.metadata; + if ( + typeof (errObj.metadata as {getMap?: unknown}).getMap === 'function' + ) { + try { + map = ( + errObj.metadata as {getMap: () => Record} + ).getMap(); + } catch { + map = errObj.metadata; + } + } + if ( + map && + typeof map === 'object' && + Object.keys(map as Record).length > 0 + ) { + return resolveResponsePayload(map); + } + } + } + } + + current = errObj.cause; + } + return undefined; +} + /** * Extracts the server-side error details message attached by GFE/backend on * server-side errors, checking the error and its `.cause` chain and falling diff --git a/core/packages/gax/test/unit/errorUtil.ts b/core/packages/gax/test/unit/errorUtil.ts index 245bc5af98d9..71c3c8eda5f7 100644 --- a/core/packages/gax/test/unit/errorUtil.ts +++ b/core/packages/gax/test/unit/errorUtil.ts @@ -52,6 +52,7 @@ import { resolveExceptionStacktrace, resolveResponseStatusCode, resolveResponsePayload, + resolveErrorResponsePayload, resolveServerExceptionDetails, recordExceptionEvent, isPreConnectionFailure, @@ -1241,6 +1242,87 @@ describe('errorUtil', () => { }); }); + describe('resolveErrorResponsePayload', () => { + it('extracts the raw HTTP response payload from translated REST errors', async () => { + const httpErr = await httpErrorWithErrorInfo(); + const payload = resolveErrorResponsePayload(httpErr); + assert.ok(payload); + assert.deepStrictEqual(JSON.parse(payload), { + error: { + code: 403, + message: 'Cloud Translation API is disabled for project 123.', + status: 'PERMISSION_DENIED', + details: [ + { + '@type': 'type.googleapis.com/google.rpc.ErrorInfo', + ...SERVICE_DISABLED_ERROR_INFO, + }, + ], + }, + }); + }); + + it('unwraps populated grpc.Metadata via getMap and omits empty Metadata', () => { + const metadata = new grpc.Metadata(); + metadata.set('x-goog-request-id', 'req-123'); + metadata.set('grpc-status-details-bin', Buffer.from([1, 2, 3])); + const grpcErr = Object.assign(new GoogleError('Unavailable'), { + code: Status.UNAVAILABLE, + metadata, + }); + assert.strictEqual( + resolveErrorResponsePayload(grpcErr), + '{"x-goog-request-id":"req-123","grpc-status-details-bin":"AQID"}', + ); + + const emptyMetaErr = Object.assign( + new GoogleError('connect ECONNREFUSED'), + { + code: Status.UNAVAILABLE, + metadata: new grpc.Metadata(), + }, + ); + assert.strictEqual(resolveErrorResponsePayload(emptyMetaErr), undefined); + assert.strictEqual( + resolveErrorResponsePayload( + Object.assign(new GoogleError('empty plain meta'), {metadata: {}}), + ), + undefined, + ); + }); + + it('supports plain-object metadata, response.data, and cause-chain traversal', () => { + const plainMetaErr = Object.assign(new GoogleError('Unavailable'), { + metadata: {'x-goog-request-id': 'req-456'}, + }); + assert.strictEqual( + resolveErrorResponsePayload(plainMetaErr), + '{"x-goog-request-id":"req-456"}', + ); + + const wrapped = new Error('outer', {cause: plainMetaErr}); + assert.strictEqual( + resolveErrorResponsePayload(wrapped), + '{"x-goog-request-id":"req-456"}', + ); + + const responseDataErr = Object.assign(new Error('HTTP 500'), { + response: {status: 500, data: {error: 'internal'}}, + }); + assert.strictEqual( + resolveErrorResponsePayload(responseDataErr), + '{"error":"internal"}', + ); + + assert.strictEqual(resolveErrorResponsePayload(undefined), undefined); + assert.strictEqual(resolveErrorResponsePayload(null), undefined); + assert.strictEqual( + resolveErrorResponsePayload(new TypeError('bad arg')), + undefined, + ); + }); + }); + describe('isPreConnectionFailure', () => { it('handles circular references in cause without infinite recursion', () => { const circularError = new GoogleError('circular error'); @@ -1455,5 +1537,13 @@ describe('errorUtil', () => { assert.strictEqual(isServerSideError(err, 'grpc'), false); assert.strictEqual(isServerSideError(err, 'http'), false); }); + + it('resolveErrorResponsePayload terminates cleanly with circular cause', () => { + const err = Object.assign(new GoogleError('test'), { + metadata: new grpc.Metadata(), + }); + err.cause = err; + assert.strictEqual(resolveErrorResponsePayload(err), undefined); + }); }); }); diff --git a/core/packages/gax/test/unit/loggerHelper.ts b/core/packages/gax/test/unit/loggerHelper.ts index a64aa8981722..ce2d491cc679 100644 --- a/core/packages/gax/test/unit/loggerHelper.ts +++ b/core/packages/gax/test/unit/loggerHelper.ts @@ -17,7 +17,9 @@ import * as assert from 'assert'; import {EventEmitter} from 'events'; import {Writable} from 'stream'; +import * as grpc from '@grpc/grpc-js'; import {context, ROOT_CONTEXT, Span, trace} from '@opentelemetry/api'; +import {GoogleAuth, PassThroughClient} from 'google-auth-library'; import {describe, it, beforeEach, afterEach} from 'mocha'; import * as sinon from 'sinon'; import * as loggingUtils from 'google-logging-utils'; @@ -25,6 +27,7 @@ import { buildLogAttributes, getGaxLogger, logCall, + logNetworkAttempt, LogPayload, } from '../../src/observability/LoggerHelper'; import {traceAttempt, traceCall} from '../../src/observability/TracerHelper'; @@ -42,10 +45,13 @@ import { ResultTuple, } from '../../src/apitypes'; import {OngoingCallPromise} from '../../src/call'; +import {GrpcClient as FallbackGrpcClient} from '../../src/fallback'; import {CallSettings} from '../../src/gax'; import {GoogleError} from '../../src/googleError'; import {Status} from '../../src/status'; +import echoProtoJson = require('../fixtures/echo.json'); import {OtelHarness} from './otelHarness'; +import {setMockFallbackError, setMockFallbackHttpResponse} from './utils'; interface CapturedLog { namespace: string; @@ -867,4 +873,470 @@ describe('LoggerHelper', () => { ); }); }); + + describe('logNetworkAttempt', () => { + it('does not log when an attempt succeeds across promise, callback, stream, or sync boundaries', async () => { + const asyncResult = await logNetworkAttempt( + dynamicArgs, + staticArgs, + async () => ({ok: true}), + ); + assert.deepStrictEqual(asyncResult, {ok: true}); + + const syncResult = logNetworkAttempt(dynamicArgs, staticArgs, () => 42); + assert.strictEqual(syncResult, 42); + + let callbackResult: unknown; + logNetworkAttempt( + dynamicArgs, + staticArgs, + loggedCallback => { + loggedCallback!(null, {cb: true}); + return undefined as unknown as ResultTuple; + }, + false, + (_err, res) => { + callbackResult = res; + }, + ); + assert.deepStrictEqual(callbackResult, {cb: true}); + + const stream = new EventEmitter(); + logNetworkAttempt(dynamicArgs, staticArgs, () => stream, true); + stream.emit('end'); + + assert.strictEqual(sink.logs.length, 0); + }); + + it('serializes populated grpc.Metadata via getMap and omits empty Metadata on pre-connection failures', async () => { + const metadata = new grpc.Metadata(); + metadata.set('x-goog-request-id', 'req-123'); + metadata.set('trace-bin', Buffer.from('abc')); + const errWithMeta = Object.assign(new GoogleError('Unavailable'), { + code: Status.UNAVAILABLE, + metadata, + }); + + await assert.rejects( + async () => { + await logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw errWithMeta; + }); + }, + (caught: unknown) => caught === errWithMeta, + ); + + assert.strictEqual(sink.logs.length, 1); + const entry = sink.logs[0]; + assert.strictEqual(entry.namespace, 'storage'); + assert.strictEqual(entry.fields.severity, 'DEBUG'); + assert.deepStrictEqual(entry.args, [ + 'RPC attempt failed: %s', + 'Unavailable', + ]); + const other = entry.fields.other as LogPayload; + assert.deepStrictEqual(other, { + 'gcp.client.service': 'storage', + 'gcp.client.version': '1.2.3', + 'gcp.client.repo': DEFAULT_GCP_REPO, + 'gcp.client.artifact': '@google-cloud/storage', + 'gcp.client.language': 'node', + 'rpc.system.name': 'grpc', + 'rpc.method': 'google.storage.v2.Storage/GetObject', + 'url.domain': 'storage.googleapis.com', + 'server.address': 'storage.googleapis.com', + 'rpc.response.status_code': 'UNAVAILABLE', + 'error.type': 'UNAVAILABLE', + 'response.payload': + '{"x-goog-request-id":"req-123","trace-bin":"YWJj"}', + }); + assert.strictEqual('exception.type' in other, false); + assert.strictEqual('exception.message' in other, false); + assert.strictEqual('exception.stacktrace' in other, false); + + sink.reset(); + const preConnectGrpcErr = Object.assign( + new GoogleError('14 UNAVAILABLE: No connection established'), + { + code: Status.UNAVAILABLE, + metadata: new grpc.Metadata(), + cause: Object.assign(new Error('connect ECONNREFUSED 127.0.0.1'), { + code: 'ECONNREFUSED', + }), + }, + ); + await assert.rejects( + async () => { + await logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw preConnectGrpcErr; + }); + }, + (caught: unknown) => caught === preConnectGrpcErr, + ); + + assert.strictEqual(sink.logs.length, 1); + const preConnectOther = sink.logs[0].fields.other as LogPayload; + assert.strictEqual('response.payload' in preConnectOther, false); + assert.strictEqual( + preConnectOther['error.type'], + 'CLIENT_CONNECTION_ERROR', + ); + assert.strictEqual('exception.type' in preConnectOther, false); + assert.strictEqual('exception.message' in preConnectOther, false); + assert.strictEqual('exception.stacktrace' in preConnectOther, false); + + sink.reset(); + const errWithoutMeta = new TypeError('client error'); + assert.throws( + () => + logNetworkAttempt(dynamicArgs, staticArgs, () => { + throw errWithoutMeta; + }), + errWithoutMeta, + ); + + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual(sink.logs[0].fields.severity, 'DEBUG'); + assert.deepStrictEqual(sink.logs[0].args, [ + 'RPC attempt failed: %s', + 'client error', + ]); + const otherWithoutMeta = sink.logs[0].fields.other as LogPayload; + assert.strictEqual('response.payload' in otherWithoutMeta, false); + assert.strictEqual('server.address' in otherWithoutMeta, false); + assert.strictEqual('exception.type' in otherWithoutMeta, false); + assert.strictEqual('exception.message' in otherWithoutMeta, false); + assert.strictEqual('exception.stacktrace' in otherWithoutMeta, false); + }); + + it('emits response.payload on HTTP REST error responses and omits it on empty or transport failures', async () => { + const httpArgs: AttemptTraceContext = { + ...dynamicArgs, + rpcType: 'http', + }; + const invokeEchoFallback = async ( + configure: (gaxGrpc: FallbackGrpcClient) => void, + ): Promise => { + const gaxGrpc = new FallbackGrpcClient({ + auth: new GoogleAuth({authClient: new PassThroughClient()}), + fallback: 'rest', + }); + const protos = gaxGrpc.loadProto(echoProtoJson); + const echoService = protos.lookupService('Echo'); + configure(gaxGrpc); + const echoStub = await gaxGrpc.createStub(echoService, { + servicePath: 'foo.example.com', + port: 443, + }); + return new Promise((resolve, reject) => { + echoStub.echo({content: 'test'}, {}, {}, (err?: Error) => { + if (err instanceof GoogleError) { + resolve(err); + } else { + reject(new Error(`Expected GoogleError, got ${err}`)); + } + }); + }); + }; + + const json503Body = JSON.stringify({ + error: { + code: 503, + message: 'Backend unavailable', + status: 'UNAVAILABLE', + }, + }); + const err503 = await invokeEchoFallback(gaxGrpc => + setMockFallbackHttpResponse( + gaxGrpc, + new Response(json503Body, {status: 503}), + ), + ); + await assert.rejects(() => + logNetworkAttempt(httpArgs, staticArgs, async () => { + throw err503; + }), + ); + assert.strictEqual(sink.logs.length, 1); + const other503 = sink.logs[0].fields.other as LogPayload; + assert.strictEqual(other503['http.response.status_code'], 503); + assert.strictEqual(other503['rpc.response.status_code'], 'UNAVAILABLE'); + assert.strictEqual(other503['response.payload'], json503Body); + assert.strictEqual('exception.type' in other503, false); + assert.strictEqual('exception.message' in other503, false); + assert.strictEqual('exception.stacktrace' in other503, false); + + sink.reset(); + const json403Body = JSON.stringify({ + error: { + code: 403, + message: 'Permission denied reading secret projects/p/secrets/s', + status: 'PERMISSION_DENIED', + }, + }); + const err403 = await invokeEchoFallback(gaxGrpc => + setMockFallbackError( + gaxGrpc, + Object.assign(new Error(json403Body), { + status: 403, + response: { + status: 403, + data: '< - See `errorRedactor` option in `gaxios` for configuration>.', + }, + }), + ), + ); + await assert.rejects(() => + logNetworkAttempt(httpArgs, staticArgs, async () => { + throw err403; + }), + ); + const other403 = sink.logs[0].fields.other as LogPayload; + assert.strictEqual(other403['http.response.status_code'], 403); + assert.strictEqual( + other403['rpc.response.status_code'], + 'PERMISSION_DENIED', + ); + assert.strictEqual(other403['response.payload'], json403Body); + + sink.reset(); + const longHtml502 = `${'x'.repeat(2048)}`; + const errHtml502 = await invokeEchoFallback(gaxGrpc => + setMockFallbackHttpResponse( + gaxGrpc, + new Response(longHtml502, {status: 502}), + ), + ); + await assert.rejects(() => + logNetworkAttempt(httpArgs, staticArgs, async () => { + throw errHtml502; + }), + ); + const otherHtml502 = sink.logs[0].fields.other as LogPayload; + assert.strictEqual(otherHtml502['response.payload'], longHtml502); + + sink.reset(); + const errEmpty502 = await invokeEchoFallback(gaxGrpc => + setMockFallbackHttpResponse(gaxGrpc, new Response(null, {status: 502})), + ); + const errEmpty403 = await invokeEchoFallback(gaxGrpc => + setMockFallbackHttpResponse(gaxGrpc, new Response(null, {status: 403})), + ); + const errTransport = await invokeEchoFallback(gaxGrpc => + setMockFallbackError( + gaxGrpc, + Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }), + ), + ); + for (const emptyErr of [errEmpty502, errEmpty403, errTransport]) { + sink.reset(); + await assert.rejects(() => + logNetworkAttempt(httpArgs, staticArgs, async () => { + throw emptyErr; + }), + ); + const otherEmpty = sink.logs[0].fields.other as LogPayload; + assert.strictEqual('response.payload' in otherEmpty, false); + } + }); + + it('captures the attempt span context and logs at most once across callback and stream failures', () => { + const stream = new EventEmitter(); + let receivedCallback: APICallback | undefined; + + traceCall(dynamicArgs, staticArgs, () => + traceAttempt(dynamicArgs, staticArgs, () => + logNetworkAttempt( + dynamicArgs, + staticArgs, + loggedCallback => { + receivedCallback = loggedCallback; + return stream; + }, + true, + () => {}, + ), + ), + ); + + const attemptErr = new GoogleError('attempt failed'); + // Simulates HTTP streaming fallback invoking both callback(err) and stream.emit('error', err). + receivedCallback!(attemptErr); + stream.emit('error', attemptErr); + + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); + const attemptSpan = spans[0]; + + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual(sink.logs[0].fields.severity, 'DEBUG'); + assert.strictEqual( + sink.logs[0].fields.telemetryTraceId, + attemptSpan.spanContext().traceId, + ); + assert.strictEqual( + sink.logs[0].fields.telemetrySpanId, + attemptSpan.spanContext().spanId, + ); + assert.deepStrictEqual(sink.logs[0].args, [ + 'RPC attempt failed: %s', + 'attempt failed', + ]); + + sink.reset(); + harness.reset(); + const streamOnly = new EventEmitter(); + streamOnly.on('error', () => {}); + traceCall(dynamicArgs, staticArgs, () => + traceAttempt(dynamicArgs, staticArgs, () => + logNetworkAttempt(dynamicArgs, staticArgs, () => streamOnly, true), + ), + ); + const streamErr = new GoogleError('stream attempt failed'); + streamOnly.emit('error', streamErr); + streamOnly.emit('error', new GoogleError('duplicate stream error')); + + const streamSpans = harness.getSpans('google-gax'); + assert.strictEqual(streamSpans.length, 2); + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual( + sink.logs[0].fields.telemetryTraceId, + streamSpans[0].spanContext().traceId, + ); + assert.strictEqual( + sink.logs[0].fields.telemetrySpanId, + streamSpans[0].spanContext().spanId, + ); + assert.deepStrictEqual(sink.logs[0].args, [ + 'RPC attempt failed: %s', + 'stream attempt failed', + ]); + }); + + it('falls back to the active outer OpenTelemetry span when GAX tracing is disabled', async () => { + const err = Object.assign(new GoogleError('Unavailable'), { + code: Status.UNAVAILABLE, + }); + const tracer = trace.getTracer('external-app'); + let outerSpan: Span | undefined; + + await assert.rejects(async () => { + await tracer.startActiveSpan('http-handler', async span => { + outerSpan = span; + try { + await logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw err; + }); + } finally { + span.end(); + } + }); + }); + + assert.ok(outerSpan); + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual( + sink.logs[0].fields.telemetryTraceId, + outerSpan.spanContext().traceId, + ); + assert.strictEqual( + sink.logs[0].fields.telemetrySpanId, + outerSpan.spanContext().spanId, + ); + + // Also preserves initialContext outer span when async callback loses active context. + sink.reset(); + let capturedCallback: APICallback | undefined; + let initialOuterSpan: Span | undefined; + tracer.startActiveSpan('lost-context-attempt', span => { + initialOuterSpan = span; + logNetworkAttempt( + dynamicArgs, + staticArgs, + loggedCallback => { + capturedCallback = loggedCallback; + return undefined as unknown as ResultTuple; + }, + false, + () => {}, + ); + span.end(); + }); + + assert.ok(initialOuterSpan); + context.with(ROOT_CONTEXT, () => { + capturedCallback!(err); + }); + + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual( + sink.logs[0].fields.telemetryTraceId, + initialOuterSpan.spanContext().traceId, + ); + assert.strictEqual( + sink.logs[0].fields.telemetrySpanId, + initialOuterSpan.spanContext().spanId, + ); + }); + + it('fails open in logNetworkAttempt and setRawHttpResponsePayload when error mutation or attribute extraction throws', async () => { + const { + getRawHttpResponsePayload, + setRawHttpResponsePayload, + } = require('../../src/googleError'); + const contextUtilModule = require('../../src/observability/contextUtil'); + + // Frozen / non-extensible GoogleError and throwing Proxy fail open in setRawHttpResponsePayload / getRawHttpResponsePayload. + const frozenErr = Object.freeze(new GoogleError('frozen')); + assert.doesNotThrow(() => + setRawHttpResponsePayload(frozenErr, '{"error":"frozen"}'), + ); + assert.strictEqual(getRawHttpResponsePayload(frozenErr), undefined); + + const hostileProxy = new Proxy(new GoogleError('proxy'), { + defineProperty() { + throw new Error('defineProperty boom'); + }, + get() { + throw new Error('get boom'); + }, + }); + assert.doesNotThrow(() => + setRawHttpResponsePayload(hostileProxy, '{"error":"proxy"}'), + ); + assert.strictEqual(getRawHttpResponsePayload(hostileProxy), undefined); + + // Throwing getActiveTraceContext or response getter inside logNetworkAttempt never masks the attempt error. + const attemptErr = Object.assign(new GoogleError('Unavailable'), { + code: Status.UNAVAILABLE, + }); + sinon + .stub(contextUtilModule, 'getActiveTraceContext') + .throws(new Error('otel context exploded')); + await assert.rejects( + () => + logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw attemptErr; + }), + (caught: unknown) => caught === attemptErr, + ); + sinon.restore(); + + const hostileResponseErr = new GoogleError('hostile response'); + Object.defineProperty(hostileResponseErr, 'response', { + get() { + throw new Error('response getter boom'); + }, + }); + await assert.rejects( + () => + logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw hostileResponseErr; + }), + (caught: unknown) => caught === hostileResponseErr, + ); + }); + }); });