From c20d5e0c5273c0a44f3c1e99b81c5eeedcb08947 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Tue, 27 Oct 2026 17:36:52 -0700 Subject: [PATCH 1/5] feat(gax): add logNetworkAttempt to LoggerHelper --- .../gax/src/observability/LoggerHelper.ts | 86 ++++++++++- core/packages/gax/test/unit/loggerHelper.ts | 145 ++++++++++++++++++ 2 files changed, 223 insertions(+), 8 deletions(-) diff --git a/core/packages/gax/src/observability/LoggerHelper.ts b/core/packages/gax/src/observability/LoggerHelper.ts index 1c15efafff07..b77cdc10ce3d 100644 --- a/core/packages/gax/src/observability/LoggerHelper.ts +++ b/core/packages/gax/src/observability/LoggerHelper.ts @@ -35,6 +35,7 @@ import { resolveExceptionMessage, resolveExceptionStacktrace, resolveExceptionType, + resolveResponsePayload, resolveResponseStatusCode, resolveRpcStatusName, } from './errorUtil'; @@ -179,6 +180,22 @@ function emitGaxLog( } } +/** + * Resolves a `LogSeverity` value across `google-logging-utils` versions where + * `LogSeverity` may be exported as a type only. + */ +function resolveLogSeverity( + level: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR', +): loggingUtils.LogSeverity { + return ( + ( + loggingUtils as { + LogSeverity?: Record; + } + ).LogSeverity?.[level] ?? (level as loggingUtils.LogSeverity) + ); +} + /** * Returns the `google-logging-utils` logger for the given GCP client service, * defaulting to the `'google-gax'` namespace when no service name is provided. @@ -391,17 +408,10 @@ export function logCall( // Build structured log attributes with exception details. const attributes = buildLogAttributes(err, dynamicArgs, staticArgs); - const warningSeverity = - ( - loggingUtils as { - LogSeverity?: {WARNING: loggingUtils.LogSeverity}; - } - ).LogSeverity?.WARNING ?? ('WARNING' as loggingUtils.LogSeverity); - // Emit structured WARNING log. logger( { - severity: warningSeverity, + severity: resolveLogSeverity('WARNING'), ...(traceId ? {telemetryTraceId: traceId} : {}), ...(spanId ? {telemetrySpanId: spanId} : {}), other: attributes, @@ -413,3 +423,63 @@ export function logCall( return executeLoggedOperation(fn, emitL3WarnLog, isStreamCall, userCallback); } + +/** + * Executes an individual RPC transport attempt and emits a `DEBUG` log entry + * if the attempt fails across promise, callback, stream, or synchronous execution boundaries. + * + * @template T + * @param {AttemptTraceContext} attemptDynamicArgs - Dynamic context for the RPC attempt. + * @param {StaticTraceContext} staticArgs - Static context for the client library. + * @param {function} fn - The transport attempt operation to execute. Receives the wrapped + * callback when `userCallback` is supplied, otherwise `undefined`. + * @param {boolean} [isStreamCall=false] - Whether the operation returns a stream (`EventEmitter`). + * @param {APICallback} [userCallback] - Optional attempt callback. + * @returns {T} The return value of `fn`. + */ +export function logNetworkAttempt( + attemptDynamicArgs: AttemptTraceContext, + staticArgs: StaticTraceContext, + fn: (loggedCallback?: APICallback) => T, + isStreamCall = false, + userCallback?: APICallback, +): T { + // Snapshot attempt context in case async rejection loses active context. + const initialContext = context.active(); + + const emitL4DebugLog = (err: unknown) => { + // Resolve logger and active T4 attempt span context for this attempt. + const logger = getGaxLogger(staticArgs?.gcpClientService); + const attemptSpan = + getActiveTraceContext()?.attemptSpan ?? + getActiveTraceContext(initialContext)?.attemptSpan; + const spanContext = attemptSpan?.spanContext(); + const traceId = spanContext?.traceId; + const spanId = spanContext?.spanId; + + // Build structured log attributes and attach attempt metadata payload. + const attributes = buildLogAttributes(err, attemptDynamicArgs, staticArgs); + const responsePayload = resolveResponsePayload( + (err as {metadata?: unknown} | null | undefined)?.metadata, + ); + if (responsePayload !== undefined) { + attributes['response.payload'] = responsePayload; + } + + // Emit structured DEBUG log for the failed attempt. + const message = resolveExceptionMessage(err, attemptDynamicArgs.rpcType); + logger( + { + severity: resolveLogSeverity('DEBUG'), + ...(traceId ? {telemetryTraceId: traceId} : {}), + ...(spanId ? {telemetrySpanId: spanId} : {}), + other: attributes, + }, + 'RPC attempt failed: %s', + message || err, + ); + }; + + // Intercept and dispatch execution boundaries for the attempt. + return executeLoggedOperation(fn, emitL4DebugLog, isStreamCall, userCallback); +} diff --git a/core/packages/gax/test/unit/loggerHelper.ts b/core/packages/gax/test/unit/loggerHelper.ts index a64aa8981722..e89a090d2cc6 100644 --- a/core/packages/gax/test/unit/loggerHelper.ts +++ b/core/packages/gax/test/unit/loggerHelper.ts @@ -25,6 +25,7 @@ import { buildLogAttributes, getGaxLogger, logCall, + logNetworkAttempt, LogPayload, } from '../../src/observability/LoggerHelper'; import {traceAttempt, traceCall} from '../../src/observability/TracerHelper'; @@ -867,4 +868,148 @@ describe('LoggerHelper', () => { ); }); }); + + describe('logNetworkAttempt', () => { + it('does not log when an attempt succeeds across promise, callback, stream, or sync boundaries', async () => { + const asyncResult = await logNetworkAttempt( + dynamicArgs, + staticArgs, + async () => ({ok: true}), + ); + assert.deepStrictEqual(asyncResult, {ok: true}); + + const syncResult = logNetworkAttempt(dynamicArgs, staticArgs, () => 42); + assert.strictEqual(syncResult, 42); + + let callbackResult: unknown; + logNetworkAttempt( + dynamicArgs, + staticArgs, + loggedCallback => { + loggedCallback!(null, {cb: true}); + return undefined as unknown as ResultTuple; + }, + false, + (_err, res) => { + callbackResult = res; + }, + ); + assert.deepStrictEqual(callbackResult, {cb: true}); + + const stream = new EventEmitter(); + logNetworkAttempt(dynamicArgs, staticArgs, () => stream, true); + stream.emit('end'); + + assert.strictEqual(sink.logs.length, 0); + }); + + it('emits a DEBUG log with response.payload when err.metadata is present and omits it when absent', async () => { + const errWithMeta = Object.assign(new GoogleError('Unavailable'), { + code: Status.UNAVAILABLE, + metadata: {'x-goog-request-id': 'req-123'}, + }); + + await assert.rejects( + async () => { + await logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw errWithMeta; + }); + }, + (caught: unknown) => caught === errWithMeta, + ); + + assert.strictEqual(sink.logs.length, 1); + const entry = sink.logs[0]; + assert.strictEqual(entry.namespace, 'storage'); + assert.strictEqual(entry.fields.severity, 'DEBUG'); + assert.deepStrictEqual(entry.args, [ + 'RPC attempt failed: %s', + 'Unavailable', + ]); + const other = entry.fields.other as LogPayload; + assert.deepStrictEqual(other, { + 'gcp.client.service': 'storage', + 'gcp.client.version': '1.2.3', + 'gcp.client.repo': DEFAULT_GCP_REPO, + 'gcp.client.artifact': '@google-cloud/storage', + 'gcp.client.language': 'node', + 'rpc.system.name': 'grpc', + 'rpc.method': 'google.storage.v2.Storage/GetObject', + 'url.domain': 'storage.googleapis.com', + 'rpc.response.status_code': 'UNAVAILABLE', + 'error.type': 'UNAVAILABLE', + 'response.payload': '{"x-goog-request-id":"req-123"}', + }); + assert.strictEqual('exception.type' in other, false); + assert.strictEqual('exception.message' in other, false); + assert.strictEqual('exception.stacktrace' in other, false); + + sink.reset(); + const errWithoutMeta = new TypeError('client error'); + assert.throws( + () => + logNetworkAttempt(dynamicArgs, staticArgs, () => { + throw errWithoutMeta; + }), + errWithoutMeta, + ); + + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual(sink.logs[0].fields.severity, 'DEBUG'); + assert.deepStrictEqual(sink.logs[0].args, [ + 'RPC attempt failed: %s', + 'client error', + ]); + const otherWithoutMeta = sink.logs[0].fields.other as LogPayload; + assert.strictEqual('response.payload' in otherWithoutMeta, false); + assert.strictEqual('exception.type' in otherWithoutMeta, false); + assert.strictEqual('exception.message' in otherWithoutMeta, false); + assert.strictEqual('exception.stacktrace' in otherWithoutMeta, false); + }); + + it('captures the attempt span context and logs on callback and stream failures', () => { + const stream = new EventEmitter(); + let receivedCallback: APICallback | undefined; + + traceCall(dynamicArgs, staticArgs, () => + traceAttempt(dynamicArgs, staticArgs, () => + logNetworkAttempt( + dynamicArgs, + staticArgs, + loggedCallback => { + receivedCallback = loggedCallback; + return stream; + }, + true, + () => {}, + ), + ), + ); + + const attemptErr = new GoogleError('attempt failed'); + receivedCallback!(attemptErr); + stream.emit('error', attemptErr); + + const spans = harness.getSpans('google-gax'); + assert.strictEqual(spans.length, 2); + const attemptSpan = spans[0]; + + assert.strictEqual(sink.logs.length, 2); + for (const logEntry of sink.logs) { + assert.strictEqual(logEntry.fields.severity, 'DEBUG'); + assert.strictEqual( + logEntry.fields.telemetryTraceId, + attemptSpan.spanContext().traceId, + ); + assert.strictEqual( + logEntry.fields.telemetrySpanId, + attemptSpan.spanContext().spanId, + ); + assert.deepStrictEqual(logEntry.args, [ + 'RPC attempt failed: %s', + 'attempt failed', + ]); + } + }); + }); }); From fc719093f45475640b4a507324dfbfa6deb4151f Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 29 Oct 2026 11:50:58 -0700 Subject: [PATCH 2/5] fix(gax): fix response.payload extraction, deduplication latch, and outer span fallback in logNetworkAttempt --- core/packages/gax/src/fallbackRest.ts | 11 +- core/packages/gax/src/fallbackServiceStub.ts | 50 ++- core/packages/gax/src/googleError.ts | 28 ++ .../gax/src/observability/LoggerHelper.ts | 32 +- .../gax/src/observability/errorUtil.ts | 102 +++++- core/packages/gax/test/unit/errorUtil.ts | 90 ++++++ core/packages/gax/test/unit/loggerHelper.ts | 299 ++++++++++++++++-- 7 files changed, 570 insertions(+), 42 deletions(-) diff --git a/core/packages/gax/src/fallbackRest.ts b/core/packages/gax/src/fallbackRest.ts index 7429924e35df..a1d5edb58efa 100644 --- a/core/packages/gax/src/fallbackRest.ts +++ b/core/packages/gax/src/fallbackRest.ts @@ -19,7 +19,7 @@ import * as serializer from 'proto3-json-serializer'; import {defaultToObjectOptions} from './fallback'; import {FetchParameters, FetchParametersMethod} from './fallbackServiceStub'; -import {GoogleError} from './googleError'; +import {GoogleError, setRawHttpResponsePayload} from './googleError'; import {rpcCodeFromHttpStatusCode} from './status'; import {getMatchedUrlTemplate, transcode} from './transcoding'; @@ -160,7 +160,7 @@ export function decodeResponse( if (ok) { throw err; } - throw _undecodableErrorResponse( + const undecodableError = _undecodableErrorResponse( rpc, httpStatusCode, `the response body is not JSON: ${ @@ -170,6 +170,8 @@ export function decodeResponse( }`, err, ); + setRawHttpResponsePayload(undecodableError, decodedString); + throw undecodableError; } if (!ok) { let error: GoogleError; @@ -179,12 +181,14 @@ export function decodeResponse( // `parseHttpError` indexes into the body, so JSON that parses but is not // an object — `null`, a string, a number — fails there for the same // reason an HTML body fails above, and must not lose the status either. - throw _undecodableErrorResponse( + const undecodableError = _undecodableErrorResponse( rpc, httpStatusCode, 'the response body is not a JSON error object', err, ); + setRawHttpResponsePayload(undecodableError, decodedString); + throw undecodableError; } // `parseHttpError` reads the status out of the response body and maps it // onto the gRPC `code`, keeping no record of the status the transport @@ -197,6 +201,7 @@ export function decodeResponse( if (httpStatusCode !== undefined) { error.httpStatusCode = httpStatusCode; } + setRawHttpResponsePayload(error, decodedString); throw error; } const message = serializer.fromProto3JSON(rpc.resolvedResponseType!, json); diff --git a/core/packages/gax/src/fallbackServiceStub.ts b/core/packages/gax/src/fallbackServiceStub.ts index cf32e5dad8cd..a4180a2ff6a0 100644 --- a/core/packages/gax/src/fallbackServiceStub.ts +++ b/core/packages/gax/src/fallbackServiceStub.ts @@ -24,7 +24,7 @@ import * as serializer from 'proto3-json-serializer'; import {isNodeJS} from './featureDetection'; import {StreamArrayParser} from './streamArrayParser'; import {defaultToObjectOptions} from './fallback'; -import {GoogleError} from './googleError'; +import {GoogleError, setRawHttpResponsePayload} from './googleError'; import {setAttemptHttpMethod} from './observability/TracerHelper'; import {rpcCodeFromHttpStatusCode, Status} from './status'; import {pipeline, PipelineSource} from 'stream'; @@ -234,6 +234,43 @@ function _parseHttpErrorFromRejection( return undefined; } +const GAXIOS_REDACTED_BODY = + '< - See `errorRedactor` option in `gaxios` for configuration>.'; + +function _isStreamLike(value: unknown): boolean { + if (!value || typeof value !== 'object') { + return false; + } + const stream = value as {pipe?: unknown; getReader?: unknown}; + return ( + typeof stream.pipe === 'function' || typeof stream.getReader === 'function' + ); +} + +function _extractRawPayloadFromRejection( + err: Error, + responseData: unknown, + httpStatus: number, +): unknown { + if ( + responseData !== undefined && + responseData !== null && + responseData !== '' && + responseData !== GAXIOS_REDACTED_BODY && + !_isStreamLike(responseData) + ) { + return responseData; + } + if ( + err.message.length > 0 && + err.message !== GAXIOS_REDACTED_BODY && + err.message !== `Request failed with status code ${httpStatus}` + ) { + return err.message; + } + return undefined; +} + function _toGoogleError(err: unknown, outcome: CallOutcome): unknown { if (err instanceof GoogleError) { return err; @@ -264,13 +301,22 @@ function _toGoogleError(err: unknown, outcome: CallOutcome): unknown { ? fetchError.status : fetchError.response?.status; if (typeof httpStatus === 'number') { + const responseData = fetchError.response?.data; const parsedError = - _parseHttpErrorFromRejection(err, fetchError.response?.data) ?? error; + _parseHttpErrorFromRejection(err, responseData) ?? error; parsedError.cause = err; if (typeof parsedError.code !== 'number') { parsedError.code = rpcCodeFromHttpStatusCode(httpStatus); } parsedError.httpStatusCode = httpStatus; + const rawPayload = _extractRawPayloadFromRejection( + err, + responseData, + httpStatus, + ); + if (rawPayload !== undefined) { + setRawHttpResponsePayload(parsedError, rawPayload); + } return parsedError; } diff --git a/core/packages/gax/src/googleError.ts b/core/packages/gax/src/googleError.ts index 961fab6bdf75..24c9120dae33 100644 --- a/core/packages/gax/src/googleError.ts +++ b/core/packages/gax/src/googleError.ts @@ -29,6 +29,34 @@ const UNKNOWN_TYPE_ENCONDED_ERROR_PREFIX = 'Unknown type encoded in'; const UNKNOWN_TYPE_NO_SUCH_TYPE = 'no such type'; const NUM_OF_PARTS_IN_PROTO_TYPE_NAME = 2; +const RAW_HTTP_RESPONSE_PAYLOAD = Symbol.for( + 'google-gax.rawHttpResponsePayload', +); + +/** + * Attaches the full raw HTTP response body non-enumerably to a GoogleError so + * telemetry debug logging can report `response.payload` without altering + * standard error formatting or serialization. + */ +export function setRawHttpResponsePayload( + err: GoogleError, + payload: unknown, +): void { + Object.defineProperty(err, RAW_HTTP_RESPONSE_PAYLOAD, { + value: payload, + configurable: true, + enumerable: false, + writable: true, + }); +} + +/** + * Reads the raw HTTP response body previously attached by + * {@link setRawHttpResponsePayload}, if any. + */ +export function getRawHttpResponsePayload(err: object): unknown { + return (err as Record)[RAW_HTTP_RESPONSE_PAYLOAD]; +} export class GoogleError extends Error { code?: Status; diff --git a/core/packages/gax/src/observability/LoggerHelper.ts b/core/packages/gax/src/observability/LoggerHelper.ts index b77cdc10ce3d..528b962a224d 100644 --- a/core/packages/gax/src/observability/LoggerHelper.ts +++ b/core/packages/gax/src/observability/LoggerHelper.ts @@ -31,11 +31,11 @@ import { isPreConnectionFailure, resolveErrorInfoDomain, resolveErrorInfoMetadataAttributes, + resolveErrorResponsePayload, resolveErrorType, resolveExceptionMessage, resolveExceptionStacktrace, resolveExceptionType, - resolveResponsePayload, resolveResponseStatusCode, resolveRpcStatusName, } from './errorUtil'; @@ -181,19 +181,15 @@ function emitGaxLog( } /** - * Resolves a `LogSeverity` value across `google-logging-utils` versions where - * `LogSeverity` may be exported as a type only. + * Resolves a `LogSeverity` enum value. Until a `google-logging-utils` release + * containing #9718 is published to npm (this repo uses `link-workspace-packages=false`), + * the published package exports `LogSeverity` as a type only while each enum member's + * runtime value equals its uppercase level string. */ function resolveLogSeverity( level: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR', ): loggingUtils.LogSeverity { - return ( - ( - loggingUtils as { - LogSeverity?: Record; - } - ).LogSeverity?.[level] ?? (level as loggingUtils.LogSeverity) - ); + return level as loggingUtils.LogSeverity; } /** @@ -448,20 +444,22 @@ export function logNetworkAttempt( const initialContext = context.active(); const emitL4DebugLog = (err: unknown) => { - // Resolve logger and active T4 attempt span context for this attempt. + // Resolve logger and active T4 attempt span (or outer OpenTelemetry span when GAX tracing is disabled). const logger = getGaxLogger(staticArgs?.gcpClientService); + const activeTrace = getActiveTraceContext(); + const initialTrace = getActiveTraceContext(initialContext); const attemptSpan = - getActiveTraceContext()?.attemptSpan ?? - getActiveTraceContext(initialContext)?.attemptSpan; + activeTrace?.attemptSpan ?? + initialTrace?.attemptSpan ?? + activeTrace?.span ?? + initialTrace?.span; const spanContext = attemptSpan?.spanContext(); const traceId = spanContext?.traceId; const spanId = spanContext?.spanId; - // Build structured log attributes and attach attempt metadata payload. + // Build structured log attributes and attach raw attempt response payload when present. const attributes = buildLogAttributes(err, attemptDynamicArgs, staticArgs); - const responsePayload = resolveResponsePayload( - (err as {metadata?: unknown} | null | undefined)?.metadata, - ); + const responsePayload = resolveErrorResponsePayload(err); if (responsePayload !== undefined) { attributes['response.payload'] = responsePayload; } diff --git a/core/packages/gax/src/observability/errorUtil.ts b/core/packages/gax/src/observability/errorUtil.ts index e8ccf2d5f8bc..dd60009d5320 100644 --- a/core/packages/gax/src/observability/errorUtil.ts +++ b/core/packages/gax/src/observability/errorUtil.ts @@ -16,7 +16,7 @@ import {Attributes, Span} from '@opentelemetry/api'; import * as protobuf from 'protobufjs'; -import {GoogleError} from '../googleError'; +import {getRawHttpResponsePayload, GoogleError} from '../googleError'; import {Status} from '../status'; /** * System and network error codes mapped to CLIENT_CONNECTION_ERROR. @@ -938,6 +938,106 @@ export function resolveResponsePayload( return bytes ? encodeBase64(bytes) : safeJsonStringify(payload); } +const GAXIOS_REDACTED_BODY = + '< - See `errorRedactor` option in `gaxios` for configuration>.'; + +function isStreamLike(value: unknown): boolean { + if (!value || typeof value !== 'object') { + return false; + } + const stream = value as {pipe?: unknown; getReader?: unknown}; + return ( + typeof stream.pipe === 'function' || typeof stream.getReader === 'function' + ); +} + +/** + * Resolves the `response.payload` attribute from a failed RPC attempt error: + * - For HTTP REST errors translated by GAX fallback, extracts the full raw HTTP + * response payload attached to the {@link GoogleError} (or a direct `response.data` + * payload when usable). + * - For gRPC errors, unwraps `Metadata#getMap()` so header values are serialized + * as scalar strings / base64 rather than single-element arrays from `Metadata#toJSON()`, + * and omits empty `Metadata` instances (such as the empty `new grpc.Metadata()` + * attached by `@grpc/grpc-js` on pre-connection failures like `ECONNREFUSED`). + * + * @param {unknown} e - The error thrown or returned by a failed attempt. + * @returns {string | undefined} The serialized response payload, or undefined if none arrived. + */ +export function resolveErrorResponsePayload(e: unknown): string | undefined { + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object' || seen.has(current)) { + break; + } + seen.add(current); + + const rawHttpPayload = getRawHttpResponsePayload(current); + if (rawHttpPayload !== undefined) { + return resolveResponsePayload(rawHttpPayload); + } + + const errObj = current as { + response?: {data?: unknown} | null; + metadata?: unknown; + cause?: unknown; + }; + + const responseData = + errObj.response && typeof errObj.response === 'object' + ? errObj.response.data + : undefined; + if ( + responseData !== undefined && + responseData !== null && + responseData !== '' && + responseData !== GAXIOS_REDACTED_BODY && + !isStreamLike(responseData) + ) { + return resolveResponsePayload(responseData); + } + + if (errObj.metadata !== undefined && errObj.metadata !== null) { + if (typeof errObj.metadata === 'string') { + if (errObj.metadata.length > 0) { + return errObj.metadata; + } + } else { + const bytes = toBytes(errObj.metadata); + if (bytes) { + if (bytes.byteLength > 0) { + return encodeBase64(bytes); + } + } else if (typeof errObj.metadata === 'object') { + let map: unknown = errObj.metadata; + if ( + typeof (errObj.metadata as {getMap?: unknown}).getMap === 'function' + ) { + try { + map = ( + errObj.metadata as {getMap: () => Record} + ).getMap(); + } catch { + map = errObj.metadata; + } + } + if ( + map && + typeof map === 'object' && + Object.keys(map as Record).length > 0 + ) { + return resolveResponsePayload(map); + } + } + } + } + + current = errObj.cause; + } + return undefined; +} + /** * Extracts the server-side error details message attached by GFE/backend on * server-side errors, checking the error and its `.cause` chain and falling diff --git a/core/packages/gax/test/unit/errorUtil.ts b/core/packages/gax/test/unit/errorUtil.ts index 245bc5af98d9..71c3c8eda5f7 100644 --- a/core/packages/gax/test/unit/errorUtil.ts +++ b/core/packages/gax/test/unit/errorUtil.ts @@ -52,6 +52,7 @@ import { resolveExceptionStacktrace, resolveResponseStatusCode, resolveResponsePayload, + resolveErrorResponsePayload, resolveServerExceptionDetails, recordExceptionEvent, isPreConnectionFailure, @@ -1241,6 +1242,87 @@ describe('errorUtil', () => { }); }); + describe('resolveErrorResponsePayload', () => { + it('extracts the raw HTTP response payload from translated REST errors', async () => { + const httpErr = await httpErrorWithErrorInfo(); + const payload = resolveErrorResponsePayload(httpErr); + assert.ok(payload); + assert.deepStrictEqual(JSON.parse(payload), { + error: { + code: 403, + message: 'Cloud Translation API is disabled for project 123.', + status: 'PERMISSION_DENIED', + details: [ + { + '@type': 'type.googleapis.com/google.rpc.ErrorInfo', + ...SERVICE_DISABLED_ERROR_INFO, + }, + ], + }, + }); + }); + + it('unwraps populated grpc.Metadata via getMap and omits empty Metadata', () => { + const metadata = new grpc.Metadata(); + metadata.set('x-goog-request-id', 'req-123'); + metadata.set('grpc-status-details-bin', Buffer.from([1, 2, 3])); + const grpcErr = Object.assign(new GoogleError('Unavailable'), { + code: Status.UNAVAILABLE, + metadata, + }); + assert.strictEqual( + resolveErrorResponsePayload(grpcErr), + '{"x-goog-request-id":"req-123","grpc-status-details-bin":"AQID"}', + ); + + const emptyMetaErr = Object.assign( + new GoogleError('connect ECONNREFUSED'), + { + code: Status.UNAVAILABLE, + metadata: new grpc.Metadata(), + }, + ); + assert.strictEqual(resolveErrorResponsePayload(emptyMetaErr), undefined); + assert.strictEqual( + resolveErrorResponsePayload( + Object.assign(new GoogleError('empty plain meta'), {metadata: {}}), + ), + undefined, + ); + }); + + it('supports plain-object metadata, response.data, and cause-chain traversal', () => { + const plainMetaErr = Object.assign(new GoogleError('Unavailable'), { + metadata: {'x-goog-request-id': 'req-456'}, + }); + assert.strictEqual( + resolveErrorResponsePayload(plainMetaErr), + '{"x-goog-request-id":"req-456"}', + ); + + const wrapped = new Error('outer', {cause: plainMetaErr}); + assert.strictEqual( + resolveErrorResponsePayload(wrapped), + '{"x-goog-request-id":"req-456"}', + ); + + const responseDataErr = Object.assign(new Error('HTTP 500'), { + response: {status: 500, data: {error: 'internal'}}, + }); + assert.strictEqual( + resolveErrorResponsePayload(responseDataErr), + '{"error":"internal"}', + ); + + assert.strictEqual(resolveErrorResponsePayload(undefined), undefined); + assert.strictEqual(resolveErrorResponsePayload(null), undefined); + assert.strictEqual( + resolveErrorResponsePayload(new TypeError('bad arg')), + undefined, + ); + }); + }); + describe('isPreConnectionFailure', () => { it('handles circular references in cause without infinite recursion', () => { const circularError = new GoogleError('circular error'); @@ -1455,5 +1537,13 @@ describe('errorUtil', () => { assert.strictEqual(isServerSideError(err, 'grpc'), false); assert.strictEqual(isServerSideError(err, 'http'), false); }); + + it('resolveErrorResponsePayload terminates cleanly with circular cause', () => { + const err = Object.assign(new GoogleError('test'), { + metadata: new grpc.Metadata(), + }); + err.cause = err; + assert.strictEqual(resolveErrorResponsePayload(err), undefined); + }); }); }); diff --git a/core/packages/gax/test/unit/loggerHelper.ts b/core/packages/gax/test/unit/loggerHelper.ts index e89a090d2cc6..6ca9bec1be91 100644 --- a/core/packages/gax/test/unit/loggerHelper.ts +++ b/core/packages/gax/test/unit/loggerHelper.ts @@ -17,7 +17,9 @@ import * as assert from 'assert'; import {EventEmitter} from 'events'; import {Writable} from 'stream'; +import * as grpc from '@grpc/grpc-js'; import {context, ROOT_CONTEXT, Span, trace} from '@opentelemetry/api'; +import {GoogleAuth, PassThroughClient} from 'google-auth-library'; import {describe, it, beforeEach, afterEach} from 'mocha'; import * as sinon from 'sinon'; import * as loggingUtils from 'google-logging-utils'; @@ -43,10 +45,13 @@ import { ResultTuple, } from '../../src/apitypes'; import {OngoingCallPromise} from '../../src/call'; +import {GrpcClient as FallbackGrpcClient} from '../../src/fallback'; import {CallSettings} from '../../src/gax'; import {GoogleError} from '../../src/googleError'; import {Status} from '../../src/status'; +import echoProtoJson = require('../fixtures/echo.json'); import {OtelHarness} from './otelHarness'; +import {setMockFallbackError, setMockFallbackHttpResponse} from './utils'; interface CapturedLog { namespace: string; @@ -903,10 +908,13 @@ describe('LoggerHelper', () => { assert.strictEqual(sink.logs.length, 0); }); - it('emits a DEBUG log with response.payload when err.metadata is present and omits it when absent', async () => { + it('serializes populated grpc.Metadata via getMap and omits empty Metadata on pre-connection failures', async () => { + const metadata = new grpc.Metadata(); + metadata.set('x-goog-request-id', 'req-123'); + metadata.set('trace-bin', Buffer.from('abc')); const errWithMeta = Object.assign(new GoogleError('Unavailable'), { code: Status.UNAVAILABLE, - metadata: {'x-goog-request-id': 'req-123'}, + metadata, }); await assert.rejects( @@ -938,12 +946,41 @@ describe('LoggerHelper', () => { 'url.domain': 'storage.googleapis.com', 'rpc.response.status_code': 'UNAVAILABLE', 'error.type': 'UNAVAILABLE', - 'response.payload': '{"x-goog-request-id":"req-123"}', + 'response.payload': + '{"x-goog-request-id":"req-123","trace-bin":"YWJj"}', }); assert.strictEqual('exception.type' in other, false); assert.strictEqual('exception.message' in other, false); assert.strictEqual('exception.stacktrace' in other, false); + sink.reset(); + const preConnectGrpcErr = Object.assign( + new GoogleError('14 UNAVAILABLE: No connection established'), + { + code: Status.UNAVAILABLE, + metadata: new grpc.Metadata(), + cause: Object.assign(new Error('connect ECONNREFUSED 127.0.0.1'), { + code: 'ECONNREFUSED', + }), + }, + ); + await assert.rejects( + async () => { + await logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw preConnectGrpcErr; + }); + }, + (caught: unknown) => caught === preConnectGrpcErr, + ); + + assert.strictEqual(sink.logs.length, 1); + const preConnectOther = sink.logs[0].fields.other as LogPayload; + assert.strictEqual('response.payload' in preConnectOther, false); + assert.strictEqual( + preConnectOther['error.type'], + 'CLIENT_CONNECTION_ERROR', + ); + sink.reset(); const errWithoutMeta = new TypeError('client error'); assert.throws( @@ -967,7 +1004,137 @@ describe('LoggerHelper', () => { assert.strictEqual('exception.stacktrace' in otherWithoutMeta, false); }); - it('captures the attempt span context and logs on callback and stream failures', () => { + it('emits response.payload on HTTP REST error responses and omits it on empty or transport failures', async () => { + const httpArgs: AttemptTraceContext = { + ...dynamicArgs, + rpcType: 'http', + }; + const invokeEchoFallback = async ( + configure: (gaxGrpc: FallbackGrpcClient) => void, + ): Promise => { + const gaxGrpc = new FallbackGrpcClient({ + auth: new GoogleAuth({authClient: new PassThroughClient()}), + fallback: 'rest', + }); + const protos = gaxGrpc.loadProto(echoProtoJson); + const echoService = protos.lookupService('Echo'); + configure(gaxGrpc); + const echoStub = await gaxGrpc.createStub(echoService, { + servicePath: 'foo.example.com', + port: 443, + }); + return new Promise((resolve, reject) => { + echoStub.echo({content: 'test'}, {}, {}, (err?: Error) => { + if (err instanceof GoogleError) { + resolve(err); + } else { + reject(new Error(`Expected GoogleError, got ${err}`)); + } + }); + }); + }; + + const json503Body = JSON.stringify({ + error: { + code: 503, + message: 'Backend unavailable', + status: 'UNAVAILABLE', + }, + }); + const err503 = await invokeEchoFallback(gaxGrpc => + setMockFallbackHttpResponse( + gaxGrpc, + new Response(json503Body, {status: 503}), + ), + ); + await assert.rejects(() => + logNetworkAttempt(httpArgs, staticArgs, async () => { + throw err503; + }), + ); + assert.strictEqual(sink.logs.length, 1); + const other503 = sink.logs[0].fields.other as LogPayload; + assert.strictEqual(other503['http.response.status_code'], 503); + assert.strictEqual(other503['rpc.response.status_code'], 'UNAVAILABLE'); + assert.strictEqual(other503['response.payload'], json503Body); + + sink.reset(); + const json403Body = JSON.stringify({ + error: { + code: 403, + message: 'Permission denied reading secret projects/p/secrets/s', + status: 'PERMISSION_DENIED', + }, + }); + const err403 = await invokeEchoFallback(gaxGrpc => + setMockFallbackError( + gaxGrpc, + Object.assign(new Error(json403Body), { + status: 403, + response: { + status: 403, + data: '< - See `errorRedactor` option in `gaxios` for configuration>.', + }, + }), + ), + ); + await assert.rejects(() => + logNetworkAttempt(httpArgs, staticArgs, async () => { + throw err403; + }), + ); + const other403 = sink.logs[0].fields.other as LogPayload; + assert.strictEqual(other403['http.response.status_code'], 403); + assert.strictEqual( + other403['rpc.response.status_code'], + 'PERMISSION_DENIED', + ); + assert.strictEqual(other403['response.payload'], json403Body); + + sink.reset(); + const longHtml502 = `${'x'.repeat(2048)}`; + const errHtml502 = await invokeEchoFallback(gaxGrpc => + setMockFallbackHttpResponse( + gaxGrpc, + new Response(longHtml502, {status: 502}), + ), + ); + await assert.rejects(() => + logNetworkAttempt(httpArgs, staticArgs, async () => { + throw errHtml502; + }), + ); + const otherHtml502 = sink.logs[0].fields.other as LogPayload; + assert.strictEqual(otherHtml502['response.payload'], longHtml502); + + sink.reset(); + const errEmpty502 = await invokeEchoFallback(gaxGrpc => + setMockFallbackHttpResponse(gaxGrpc, new Response(null, {status: 502})), + ); + const errEmpty403 = await invokeEchoFallback(gaxGrpc => + setMockFallbackHttpResponse(gaxGrpc, new Response(null, {status: 403})), + ); + const errTransport = await invokeEchoFallback(gaxGrpc => + setMockFallbackError( + gaxGrpc, + Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }), + ), + ); + for (const emptyErr of [errEmpty502, errEmpty403, errTransport]) { + sink.reset(); + await assert.rejects(() => + logNetworkAttempt(httpArgs, staticArgs, async () => { + throw emptyErr; + }), + ); + const otherEmpty = sink.logs[0].fields.other as LogPayload; + assert.strictEqual('response.payload' in otherEmpty, false); + } + }); + + it('captures the attempt span context and logs at most once across callback and stream failures', () => { const stream = new EventEmitter(); let receivedCallback: APICallback | undefined; @@ -987,6 +1154,7 @@ describe('LoggerHelper', () => { ); const attemptErr = new GoogleError('attempt failed'); + // Simulates HTTP streaming fallback invoking both callback(err) and stream.emit('error', err). receivedCallback!(attemptErr); stream.emit('error', attemptErr); @@ -994,22 +1162,115 @@ describe('LoggerHelper', () => { assert.strictEqual(spans.length, 2); const attemptSpan = spans[0]; - assert.strictEqual(sink.logs.length, 2); - for (const logEntry of sink.logs) { - assert.strictEqual(logEntry.fields.severity, 'DEBUG'); - assert.strictEqual( - logEntry.fields.telemetryTraceId, - attemptSpan.spanContext().traceId, - ); - assert.strictEqual( - logEntry.fields.telemetrySpanId, - attemptSpan.spanContext().spanId, + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual(sink.logs[0].fields.severity, 'DEBUG'); + assert.strictEqual( + sink.logs[0].fields.telemetryTraceId, + attemptSpan.spanContext().traceId, + ); + assert.strictEqual( + sink.logs[0].fields.telemetrySpanId, + attemptSpan.spanContext().spanId, + ); + assert.deepStrictEqual(sink.logs[0].args, [ + 'RPC attempt failed: %s', + 'attempt failed', + ]); + + sink.reset(); + harness.reset(); + const streamOnly = new EventEmitter(); + streamOnly.on('error', () => {}); + traceCall(dynamicArgs, staticArgs, () => + traceAttempt(dynamicArgs, staticArgs, () => + logNetworkAttempt(dynamicArgs, staticArgs, () => streamOnly, true), + ), + ); + const streamErr = new GoogleError('stream attempt failed'); + streamOnly.emit('error', streamErr); + streamOnly.emit('error', new GoogleError('duplicate stream error')); + + const streamSpans = harness.getSpans('google-gax'); + assert.strictEqual(streamSpans.length, 2); + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual( + sink.logs[0].fields.telemetryTraceId, + streamSpans[0].spanContext().traceId, + ); + assert.strictEqual( + sink.logs[0].fields.telemetrySpanId, + streamSpans[0].spanContext().spanId, + ); + assert.deepStrictEqual(sink.logs[0].args, [ + 'RPC attempt failed: %s', + 'stream attempt failed', + ]); + }); + + it('falls back to the active outer OpenTelemetry span when GAX tracing is disabled', async () => { + const err = Object.assign(new GoogleError('Unavailable'), { + code: Status.UNAVAILABLE, + }); + const tracer = trace.getTracer('external-app'); + let outerSpan: Span | undefined; + + await assert.rejects(async () => { + await tracer.startActiveSpan('http-handler', async span => { + outerSpan = span; + try { + await logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw err; + }); + } finally { + span.end(); + } + }); + }); + + assert.ok(outerSpan); + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual( + sink.logs[0].fields.telemetryTraceId, + outerSpan.spanContext().traceId, + ); + assert.strictEqual( + sink.logs[0].fields.telemetrySpanId, + outerSpan.spanContext().spanId, + ); + + // Also preserves initialContext outer span when async callback loses active context. + sink.reset(); + let capturedCallback: APICallback | undefined; + let initialOuterSpan: Span | undefined; + tracer.startActiveSpan('lost-context-attempt', span => { + initialOuterSpan = span; + logNetworkAttempt( + dynamicArgs, + staticArgs, + loggedCallback => { + capturedCallback = loggedCallback; + return undefined as unknown as ResultTuple; + }, + false, + () => {}, ); - assert.deepStrictEqual(logEntry.args, [ - 'RPC attempt failed: %s', - 'attempt failed', - ]); - } + span.end(); + }); + + assert.ok(initialOuterSpan); + context.with(ROOT_CONTEXT, () => { + capturedCallback!(err); + }); + + assert.strictEqual(sink.logs.length, 1); + assert.strictEqual( + sink.logs[0].fields.telemetryTraceId, + initialOuterSpan.spanContext().traceId, + ); + assert.strictEqual( + sink.logs[0].fields.telemetrySpanId, + initialOuterSpan.spanContext().spanId, + ); }); }); }); From 379bfa79ca7df48e7887b50dcd68a79e906bf9c1 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 29 Oct 2026 12:43:52 -0700 Subject: [PATCH 3/5] fix(gax): reuse buildLogAttributes exception attributes in logNetworkAttempt --- .../gax/src/observability/LoggerHelper.ts | 3 +-- core/packages/gax/test/unit/loggerHelper.ts | 17 +++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/core/packages/gax/src/observability/LoggerHelper.ts b/core/packages/gax/src/observability/LoggerHelper.ts index 528b962a224d..d1e21af19d28 100644 --- a/core/packages/gax/src/observability/LoggerHelper.ts +++ b/core/packages/gax/src/observability/LoggerHelper.ts @@ -465,7 +465,6 @@ export function logNetworkAttempt( } // Emit structured DEBUG log for the failed attempt. - const message = resolveExceptionMessage(err, attemptDynamicArgs.rpcType); logger( { severity: resolveLogSeverity('DEBUG'), @@ -474,7 +473,7 @@ export function logNetworkAttempt( other: attributes, }, 'RPC attempt failed: %s', - message || err, + attributes['exception.message'] || err, ); }; diff --git a/core/packages/gax/test/unit/loggerHelper.ts b/core/packages/gax/test/unit/loggerHelper.ts index 6ca9bec1be91..36bf95ccdd27 100644 --- a/core/packages/gax/test/unit/loggerHelper.ts +++ b/core/packages/gax/test/unit/loggerHelper.ts @@ -944,14 +944,15 @@ describe('LoggerHelper', () => { 'rpc.system.name': 'grpc', 'rpc.method': 'google.storage.v2.Storage/GetObject', 'url.domain': 'storage.googleapis.com', + 'server.address': 'storage.googleapis.com', 'rpc.response.status_code': 'UNAVAILABLE', 'error.type': 'UNAVAILABLE', + 'exception.type': 'GoogleError', + 'exception.message': 'Unavailable', + 'exception.stacktrace': errWithMeta.stack, 'response.payload': '{"x-goog-request-id":"req-123","trace-bin":"YWJj"}', }); - assert.strictEqual('exception.type' in other, false); - assert.strictEqual('exception.message' in other, false); - assert.strictEqual('exception.stacktrace' in other, false); sink.reset(); const preConnectGrpcErr = Object.assign( @@ -999,9 +1000,13 @@ describe('LoggerHelper', () => { ]); const otherWithoutMeta = sink.logs[0].fields.other as LogPayload; assert.strictEqual('response.payload' in otherWithoutMeta, false); - assert.strictEqual('exception.type' in otherWithoutMeta, false); - assert.strictEqual('exception.message' in otherWithoutMeta, false); - assert.strictEqual('exception.stacktrace' in otherWithoutMeta, false); + assert.strictEqual('server.address' in otherWithoutMeta, false); + assert.strictEqual(otherWithoutMeta['exception.type'], 'TypeError'); + assert.strictEqual(otherWithoutMeta['exception.message'], 'client error'); + assert.strictEqual( + otherWithoutMeta['exception.stacktrace'], + errWithoutMeta.stack, + ); }); it('emits response.payload on HTTP REST error responses and omits it on empty or transport failures', async () => { From 18bcbd13013f8f2ee205a430ddf17b7921585836 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 29 Oct 2026 20:44:50 -0700 Subject: [PATCH 4/5] fix(gax): make setRawHttpResponsePayload and logNetworkAttempt fail-open --- core/packages/gax/src/googleError.ts | 22 ++++--- .../gax/src/observability/LoggerHelper.ts | 11 +++- core/packages/gax/test/unit/loggerHelper.ts | 58 +++++++++++++++++++ 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/core/packages/gax/src/googleError.ts b/core/packages/gax/src/googleError.ts index 24c9120dae33..58e6a11507d8 100644 --- a/core/packages/gax/src/googleError.ts +++ b/core/packages/gax/src/googleError.ts @@ -42,12 +42,16 @@ export function setRawHttpResponsePayload( err: GoogleError, payload: unknown, ): void { - Object.defineProperty(err, RAW_HTTP_RESPONSE_PAYLOAD, { - value: payload, - configurable: true, - enumerable: false, - writable: true, - }); + try { + Object.defineProperty(err, RAW_HTTP_RESPONSE_PAYLOAD, { + value: payload, + configurable: true, + enumerable: false, + writable: true, + }); + } catch { + // Fail open if err is frozen, non-extensible, or traps defineProperty. + } } /** @@ -55,7 +59,11 @@ export function setRawHttpResponsePayload( * {@link setRawHttpResponsePayload}, if any. */ export function getRawHttpResponsePayload(err: object): unknown { - return (err as Record)[RAW_HTTP_RESPONSE_PAYLOAD]; + try { + return (err as Record)[RAW_HTTP_RESPONSE_PAYLOAD]; + } catch { + return undefined; + } } export class GoogleError extends Error { diff --git a/core/packages/gax/src/observability/LoggerHelper.ts b/core/packages/gax/src/observability/LoggerHelper.ts index d1e21af19d28..1cd49065ae65 100644 --- a/core/packages/gax/src/observability/LoggerHelper.ts +++ b/core/packages/gax/src/observability/LoggerHelper.ts @@ -441,13 +441,20 @@ export function logNetworkAttempt( userCallback?: APICallback, ): T { // Snapshot attempt context in case async rejection loses active context. - const initialContext = context.active(); + let initialContext: ReturnType | undefined; + try { + initialContext = context.active(); + } catch { + // Fail open if active context retrieval throws. + } const emitL4DebugLog = (err: unknown) => { // Resolve logger and active T4 attempt span (or outer OpenTelemetry span when GAX tracing is disabled). const logger = getGaxLogger(staticArgs?.gcpClientService); const activeTrace = getActiveTraceContext(); - const initialTrace = getActiveTraceContext(initialContext); + const initialTrace = initialContext + ? getActiveTraceContext(initialContext) + : undefined; const attemptSpan = activeTrace?.attemptSpan ?? initialTrace?.attemptSpan ?? diff --git a/core/packages/gax/test/unit/loggerHelper.ts b/core/packages/gax/test/unit/loggerHelper.ts index 36bf95ccdd27..e953a589f19b 100644 --- a/core/packages/gax/test/unit/loggerHelper.ts +++ b/core/packages/gax/test/unit/loggerHelper.ts @@ -1277,5 +1277,63 @@ describe('LoggerHelper', () => { initialOuterSpan.spanContext().spanId, ); }); + + it('fails open in logNetworkAttempt and setRawHttpResponsePayload when error mutation or attribute extraction throws', async () => { + const { + getRawHttpResponsePayload, + setRawHttpResponsePayload, + } = require('../../src/googleError'); + const contextUtilModule = require('../../src/observability/contextUtil'); + + // Frozen / non-extensible GoogleError and throwing Proxy fail open in setRawHttpResponsePayload / getRawHttpResponsePayload. + const frozenErr = Object.freeze(new GoogleError('frozen')); + assert.doesNotThrow(() => + setRawHttpResponsePayload(frozenErr, '{"error":"frozen"}'), + ); + assert.strictEqual(getRawHttpResponsePayload(frozenErr), undefined); + + const hostileProxy = new Proxy(new GoogleError('proxy'), { + defineProperty() { + throw new Error('defineProperty boom'); + }, + get() { + throw new Error('get boom'); + }, + }); + assert.doesNotThrow(() => + setRawHttpResponsePayload(hostileProxy, '{"error":"proxy"}'), + ); + assert.strictEqual(getRawHttpResponsePayload(hostileProxy), undefined); + + // Throwing getActiveTraceContext or response getter inside logNetworkAttempt never masks the attempt error. + const attemptErr = Object.assign(new GoogleError('Unavailable'), { + code: Status.UNAVAILABLE, + }); + sinon + .stub(contextUtilModule, 'getActiveTraceContext') + .throws(new Error('otel context exploded')); + await assert.rejects( + () => + logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw attemptErr; + }), + (caught: unknown) => caught === attemptErr, + ); + sinon.restore(); + + const hostileResponseErr = new GoogleError('hostile response'); + Object.defineProperty(hostileResponseErr, 'response', { + get() { + throw new Error('response getter boom'); + }, + }); + await assert.rejects( + () => + logNetworkAttempt(dynamicArgs, staticArgs, async () => { + throw hostileResponseErr; + }), + (caught: unknown) => caught === hostileResponseErr, + ); + }); }); }); From c1753818ad765496beccd136fb7ee9597076b52b Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Thu, 29 Oct 2026 20:49:46 -0700 Subject: [PATCH 5/5] fix(gax): omit exception attributes on L4 logNetworkAttempt logs --- .../gax/src/observability/LoggerHelper.ts | 33 ++++++++++++------- core/packages/gax/test/unit/loggerHelper.ts | 21 +++++++----- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/core/packages/gax/src/observability/LoggerHelper.ts b/core/packages/gax/src/observability/LoggerHelper.ts index 1cd49065ae65..c1c09ce3703a 100644 --- a/core/packages/gax/src/observability/LoggerHelper.ts +++ b/core/packages/gax/src/observability/LoggerHelper.ts @@ -229,12 +229,15 @@ export function getGaxLogger( * @param {unknown} err - The error thrown or reported for the call. * @param {AttemptTraceContext} dynamicArgs - Dynamic context for the RPC call or attempt. * @param {StaticTraceContext} [staticArgs={}] - Static context for the client library. + * @param {boolean} [includeExceptionAttributes=true] - Whether to populate `exception.*` + * attributes (set on L3 client call logs, omitted on L4 transport attempt logs). * @returns {LogPayload} The populated log payload attributes with undefined fields omitted. */ export function buildLogAttributes( err: unknown, dynamicArgs: AttemptTraceContext, staticArgs: StaticTraceContext = {}, + includeExceptionAttributes = true, ): LogPayload { const {rawAddress, urlDomain} = resolveEndpointAndDomain( dynamicArgs, @@ -288,14 +291,16 @@ export function buildLogAttributes( } Object.assign(attributes, resolveErrorInfoMetadataAttributes(err)); - attributes['exception.type'] = resolveExceptionType(err); - attributes['exception.message'] = resolveExceptionMessage( - err, - dynamicArgs.rpcType, - ); - const stacktrace = resolveExceptionStacktrace(err); - if (stacktrace !== undefined) { - attributes['exception.stacktrace'] = stacktrace; + if (includeExceptionAttributes) { + attributes['exception.type'] = resolveExceptionType(err); + attributes['exception.message'] = resolveExceptionMessage( + err, + dynamicArgs.rpcType, + ); + const stacktrace = resolveExceptionStacktrace(err); + if (stacktrace !== undefined) { + attributes['exception.stacktrace'] = stacktrace; + } } return attributes; @@ -464,14 +469,20 @@ export function logNetworkAttempt( const traceId = spanContext?.traceId; const spanId = spanContext?.spanId; - // Build structured log attributes and attach raw attempt response payload when present. - const attributes = buildLogAttributes(err, attemptDynamicArgs, staticArgs); + // Build structured log attributes without exception details and attach raw attempt response payload when present. + const attributes = buildLogAttributes( + err, + attemptDynamicArgs, + staticArgs, + false, + ); const responsePayload = resolveErrorResponsePayload(err); if (responsePayload !== undefined) { attributes['response.payload'] = responsePayload; } // Emit structured DEBUG log for the failed attempt. + const message = resolveExceptionMessage(err, attemptDynamicArgs.rpcType); logger( { severity: resolveLogSeverity('DEBUG'), @@ -480,7 +491,7 @@ export function logNetworkAttempt( other: attributes, }, 'RPC attempt failed: %s', - attributes['exception.message'] || err, + message || err, ); }; diff --git a/core/packages/gax/test/unit/loggerHelper.ts b/core/packages/gax/test/unit/loggerHelper.ts index e953a589f19b..ce2d491cc679 100644 --- a/core/packages/gax/test/unit/loggerHelper.ts +++ b/core/packages/gax/test/unit/loggerHelper.ts @@ -947,12 +947,12 @@ describe('LoggerHelper', () => { 'server.address': 'storage.googleapis.com', 'rpc.response.status_code': 'UNAVAILABLE', 'error.type': 'UNAVAILABLE', - 'exception.type': 'GoogleError', - 'exception.message': 'Unavailable', - 'exception.stacktrace': errWithMeta.stack, 'response.payload': '{"x-goog-request-id":"req-123","trace-bin":"YWJj"}', }); + assert.strictEqual('exception.type' in other, false); + assert.strictEqual('exception.message' in other, false); + assert.strictEqual('exception.stacktrace' in other, false); sink.reset(); const preConnectGrpcErr = Object.assign( @@ -981,6 +981,9 @@ describe('LoggerHelper', () => { preConnectOther['error.type'], 'CLIENT_CONNECTION_ERROR', ); + assert.strictEqual('exception.type' in preConnectOther, false); + assert.strictEqual('exception.message' in preConnectOther, false); + assert.strictEqual('exception.stacktrace' in preConnectOther, false); sink.reset(); const errWithoutMeta = new TypeError('client error'); @@ -1001,12 +1004,9 @@ describe('LoggerHelper', () => { const otherWithoutMeta = sink.logs[0].fields.other as LogPayload; assert.strictEqual('response.payload' in otherWithoutMeta, false); assert.strictEqual('server.address' in otherWithoutMeta, false); - assert.strictEqual(otherWithoutMeta['exception.type'], 'TypeError'); - assert.strictEqual(otherWithoutMeta['exception.message'], 'client error'); - assert.strictEqual( - otherWithoutMeta['exception.stacktrace'], - errWithoutMeta.stack, - ); + assert.strictEqual('exception.type' in otherWithoutMeta, false); + assert.strictEqual('exception.message' in otherWithoutMeta, false); + assert.strictEqual('exception.stacktrace' in otherWithoutMeta, false); }); it('emits response.payload on HTTP REST error responses and omits it on empty or transport failures', async () => { @@ -1062,6 +1062,9 @@ describe('LoggerHelper', () => { assert.strictEqual(other503['http.response.status_code'], 503); assert.strictEqual(other503['rpc.response.status_code'], 'UNAVAILABLE'); assert.strictEqual(other503['response.payload'], json503Body); + assert.strictEqual('exception.type' in other503, false); + assert.strictEqual('exception.message' in other503, false); + assert.strictEqual('exception.stacktrace' in other503, false); sink.reset(); const json403Body = JSON.stringify({