From 134fd58037b0f8bb6478f5677216addf54d2e39a Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Tue, 27 Oct 2026 11:09:10 -0700 Subject: [PATCH 1/3] feat(gax): add resolveResponsePayload helper for response.payload Add resolveResponsePayload to errorUtil so LoggerHelper can populate response.payload with the full response serialized as JSON (string payloads are returned as-is). It prefers the raw response gax provides for paginated and long-running calls over the processed response, and returns undefined when there is no response. Payloads are not redacted, so a TODO(security) notes they must only be emitted when logging is explicitly enabled. --- .../gax/src/observability/errorUtil.ts | 26 +++++++++ core/packages/gax/test/unit/errorUtil.ts | 57 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/core/packages/gax/src/observability/errorUtil.ts b/core/packages/gax/src/observability/errorUtil.ts index ce07114783ce..d517a129093d 100644 --- a/core/packages/gax/src/observability/errorUtil.ts +++ b/core/packages/gax/src/observability/errorUtil.ts @@ -845,6 +845,32 @@ export function safeJsonStringify(value: unknown): string | undefined { } } +/** + * Resolves the `response.payload` attribute: the full response payload of a call, + * serialized as JSON (string payloads are returned as-is) without truncation or + * redaction. Prefers `rawResponse`, the unprocessed response gax provides for + * paginated and long-running calls (the full page response or the raw Operation), + * over the processed `response`. + * + * @param {unknown} response - The response passed to the callback or resolved by the promise. + * @param {unknown} [rawResponse] - The raw (unprocessed) response, when gax provides one. + * @returns {string | undefined} The serialized payload, or undefined if there is no response. + */ +export function resolveResponsePayload( + response: unknown, + rawResponse?: unknown, +): string | undefined { + const payload = rawResponse ?? response; + if (payload === undefined || payload === null) { + return undefined; + } + // TODO(security): The payload is returned in full because `response.payload` is defined + // as the raw response, so it can contain sensitive data (e.g. secret values or access + // tokens). Only emit it when logging is explicitly enabled (see `checkLoggingEnabled`), + // and consider redacting known sensitive fields. + return typeof payload === 'string' ? payload : safeJsonStringify(payload); +} + /** * Extracts the server-side error details message attached by GFE/backend on * server-side errors, checking the error and its `.cause` chain and falling diff --git a/core/packages/gax/test/unit/errorUtil.ts b/core/packages/gax/test/unit/errorUtil.ts index e390e2223034..6c7f324c843b 100644 --- a/core/packages/gax/test/unit/errorUtil.ts +++ b/core/packages/gax/test/unit/errorUtil.ts @@ -51,6 +51,7 @@ import { resolveExceptionMessage, resolveExceptionStacktrace, resolveResponseStatusCode, + resolveResponsePayload, resolveServerExceptionDetails, recordExceptionEvent, isPreConnectionFailure, @@ -1123,6 +1124,62 @@ describe('errorUtil', () => { }); }); + describe('resolveResponsePayload', () => { + it('serializes the full response as JSON and returns string payloads as-is', () => { + assert.strictEqual( + resolveResponsePayload({ + name: 'projects/p/secrets/s', + labels: {env: 'prod'}, + versions: [1, 2], + }), + '{"name":"projects/p/secrets/s","labels":{"env":"prod"},"versions":[1,2]}', + ); + assert.strictEqual(resolveResponsePayload('raw body'), 'raw body'); + assert.strictEqual(resolveResponsePayload({}), '{}'); + assert.strictEqual(resolveResponsePayload([]), '[]'); + }); + + it('prefers the raw response provided for paginated and long-running calls', () => { + const resources = [{name: 'a'}, {name: 'b'}]; + const rawResponse = {items: resources, nextPageToken: 'token'}; + assert.strictEqual( + resolveResponsePayload(resources, rawResponse), + '{"items":[{"name":"a"},{"name":"b"}],"nextPageToken":"token"}', + ); + // A null raw response falls back to the response. + assert.strictEqual( + resolveResponsePayload({name: 'a'}, null), + '{"name":"a"}', + ); + }); + + it('returns undefined when there is no response', () => { + assert.strictEqual(resolveResponsePayload(undefined), undefined); + assert.strictEqual(resolveResponsePayload(null), undefined); + assert.strictEqual(resolveResponsePayload(null, null), undefined); + }); + + it('serializes protobuf messages through their JSON representation', () => { + const message = new GoogleErrorDecoder().root + .lookupType('google.rpc.ErrorInfo') + .fromObject(SERVICE_DISABLED_ERROR_INFO); + assert.strictEqual( + resolveResponsePayload(message), + '{"reason":"SERVICE_DISABLED","domain":"googleapis.com",' + + '"metadata":{"service":"translate.googleapis.com","consumer":"projects/123"}}', + ); + }); + + it('handles circular references and BigInt values without throwing', () => { + const response: Record = {size: 10n}; + response.self = response; + assert.strictEqual( + resolveResponsePayload(response), + '{"size":"10","self":"[Circular]"}', + ); + }); + }); + describe('isPreConnectionFailure', () => { it('handles circular references in cause without infinite recursion', () => { const circularError = new GoogleError('circular error'); From 38f3146b311f162b4bc1de2b072358eb32b57e1d Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Tue, 27 Oct 2026 11:44:49 -0700 Subject: [PATCH 2/3] fix(gax): base64-encode binary data in response.payload Buffers and other binary data (Uint8Array, ArrayBuffer) were serialized one JSON entry per byte, e.g. {"type":"Buffer","data":[...]} or {"0":123,...}, bloating logs. Binary payloads are now base64 encoded, as are binary values nested in a response such as bytes fields. Base64 is how the protobuf JSON mapping represents bytes and matches what REST responses already emit. Uses Buffer when available and protobufjs's pure JavaScript encoder otherwise (e.g. in browsers). --- .../gax/src/observability/errorUtil.ts | 69 +++++++++++++++++-- core/packages/gax/test/unit/errorUtil.ts | 57 +++++++++++++++ 2 files changed, 119 insertions(+), 7 deletions(-) diff --git a/core/packages/gax/src/observability/errorUtil.ts b/core/packages/gax/src/observability/errorUtil.ts index d517a129093d..b54bc3a7276c 100644 --- a/core/packages/gax/src/observability/errorUtil.ts +++ b/core/packages/gax/src/observability/errorUtil.ts @@ -15,6 +15,7 @@ */ import {Attributes, Span} from '@opentelemetry/api'; +import * as protobuf from 'protobufjs'; import {GoogleError} from '../googleError'; import {Status} from '../status'; /** @@ -798,9 +799,46 @@ export function isServerSideError( return resolveServerErrorCode(e, rpcType) !== undefined; } +/** + * Returns the bytes of binary data: an ArrayBuffer, or a view of one such as a Buffer + * or Uint8Array. + * + * @param {unknown} value - The value to check. + * @returns {Uint8Array | undefined} The bytes of `value`, or undefined if it is not binary data. + */ +function toBytes(value: unknown): Uint8Array | undefined { + if (ArrayBuffer.isView(value)) { + return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + } + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + return undefined; +} + +/** + * Encodes bytes as base64, using Buffer when it is available and a pure JavaScript + * encoder otherwise (e.g. in browsers). + * + * @param {Uint8Array} bytes - The bytes to encode. + * @returns {string} The base64 encoding of `bytes`. + */ +function encodeBase64(bytes: Uint8Array): string { + if (typeof Buffer !== 'undefined') { + return Buffer.from( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength, + ).toString('base64'); + } + return protobuf.util.base64.encode(bytes, 0, bytes.length); +} + /** * Safely converts a value to a JSON string without throwing exceptions on - * circular references, BigInt values, or non-serializable properties. + * circular references, BigInt values, or non-serializable properties. Binary data + * (e.g. a Buffer or Uint8Array) is serialized as a base64 string rather than one + * JSON entry per byte. * * @param {unknown} value - The value to serialize. * @returns {string | undefined} The serialized JSON string, or undefined if `value` is undefined or unserializable. @@ -813,13 +851,24 @@ export function safeJsonStringify(value: unknown): string | undefined { const ancestors: unknown[] = []; return JSON.stringify( value, - function (this: unknown, _key: string, val: unknown) { + function (this: unknown, key: string, val: unknown) { if (typeof val === 'bigint') { return val.toString(); } if (typeof val !== 'object' || val === null) { return val; } + // Encode binary data as base64. Buffer#toJSON runs before this replacer and + // turns a Buffer into {type: 'Buffer', data: [...]}, so in that case the + // original Buffer is read back from the holder. + const bytes = toBytes( + (val as {type?: unknown}).type === 'Buffer' + ? (this as Record)[key] + : val, + ); + if (bytes) { + return encodeBase64(bytes); + } // Track object ancestry to replace circular references. if (ancestors.includes(this)) { while ( @@ -847,10 +896,11 @@ export function safeJsonStringify(value: unknown): string | undefined { /** * Resolves the `response.payload` attribute: the full response payload of a call, - * serialized as JSON (string payloads are returned as-is) without truncation or - * redaction. Prefers `rawResponse`, the unprocessed response gax provides for - * paginated and long-running calls (the full page response or the raw Operation), - * over the processed `response`. + * serialized as JSON without truncation or redaction. String payloads are returned + * as-is and binary payloads (e.g. a Buffer or Uint8Array) as base64, as are binary + * values nested in the response (e.g. `bytes` fields). Prefers `rawResponse`, the + * unprocessed response gax provides for paginated and long-running calls (the full + * page response or the raw Operation), over the processed `response`. * * @param {unknown} response - The response passed to the callback or resolved by the promise. * @param {unknown} [rawResponse] - The raw (unprocessed) response, when gax provides one. @@ -868,7 +918,12 @@ export function resolveResponsePayload( // as the raw response, so it can contain sensitive data (e.g. secret values or access // tokens). Only emit it when logging is explicitly enabled (see `checkLoggingEnabled`), // and consider redacting known sensitive fields. - return typeof payload === 'string' ? payload : safeJsonStringify(payload); + if (typeof payload === 'string') { + return payload; + } + // Binary payloads are base64 encoded and, like strings, not wrapped in JSON quotes. + const bytes = toBytes(payload); + return bytes ? encodeBase64(bytes) : safeJsonStringify(payload); } /** diff --git a/core/packages/gax/test/unit/errorUtil.ts b/core/packages/gax/test/unit/errorUtil.ts index 6c7f324c843b..994ac8308f96 100644 --- a/core/packages/gax/test/unit/errorUtil.ts +++ b/core/packages/gax/test/unit/errorUtil.ts @@ -1122,6 +1122,23 @@ describe('errorUtil', () => { }; assert.strictEqual(safeJsonStringify(badObj), undefined); }); + + it('serializes binary data as base64 instead of one entry per byte', () => { + // Buffers reach the replacer already converted by Buffer#toJSON, unlike other + // binary data such as Uint8Arrays, so both are covered. + assert.strictEqual( + safeJsonStringify({ + buffer: Buffer.from('hello'), + list: [new Uint8Array([1, 2, 3])], + }), + '{"buffer":"aGVsbG8=","list":["AQID"]}', + ); + // Objects that only look like a serialized Buffer are left as-is. + assert.strictEqual( + safeJsonStringify({type: 'Buffer', data: [1, 2]}), + '{"type":"Buffer","data":[1,2]}', + ); + }); }); describe('resolveResponsePayload', () => { @@ -1178,6 +1195,46 @@ describe('errorUtil', () => { '{"size":"10","self":"[Circular]"}', ); }); + + it('encodes binary payloads as base64', () => { + assert.strictEqual( + resolveResponsePayload(Buffer.from('hello')), + 'aGVsbG8=', + ); + assert.strictEqual( + resolveResponsePayload(new Uint8Array([1, 2, 3]).buffer), + 'AQID', + ); + // Only the viewed bytes are encoded, not the rest of the underlying ArrayBuffer. + assert.strictEqual( + resolveResponsePayload(new Uint8Array([0, 1, 2, 3, 4]).subarray(1, 4)), + 'AQID', + ); + // Typed arrays from another realm are recognized too. + assert.strictEqual( + resolveResponsePayload(vm.runInNewContext('new Uint8Array([1, 2, 3])')), + 'AQID', + ); + }); + + it('encodes binary payloads as base64 when Buffer is not defined', () => { + const originalBuffer = (globalThis as Record).Buffer; + try { + delete (globalThis as Record).Buffer; + assert.strictEqual( + resolveResponsePayload( + new Uint8Array([0, 1, 2, 3, 4]).subarray(1, 4), + ), + 'AQID', + ); + assert.strictEqual( + resolveResponsePayload(new Uint8Array([255])), + '/w==', + ); + } finally { + (globalThis as Record).Buffer = originalBuffer; + } + }); }); describe('isPreConnectionFailure', () => { From 36e6a90e9ca04834bb520d95fe0815430b9580e9 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Tue, 27 Oct 2026 11:50:44 -0700 Subject: [PATCH 3/3] test(gax): remove resolveResponsePayload tests duplicated by safeJsonStringify tests resolveResponsePayload passes non-string, non-binary payloads to safeJsonStringify, whose own tests cover circular references and BigInt values. Also drop assertions that repeat the JSON path ([]) or combine two existing cases ((null, null)). --- core/packages/gax/test/unit/errorUtil.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/core/packages/gax/test/unit/errorUtil.ts b/core/packages/gax/test/unit/errorUtil.ts index 994ac8308f96..205d485b87d2 100644 --- a/core/packages/gax/test/unit/errorUtil.ts +++ b/core/packages/gax/test/unit/errorUtil.ts @@ -1152,8 +1152,8 @@ describe('errorUtil', () => { '{"name":"projects/p/secrets/s","labels":{"env":"prod"},"versions":[1,2]}', ); assert.strictEqual(resolveResponsePayload('raw body'), 'raw body'); + // Empty responses (e.g. google.protobuf.Empty) still have a payload. assert.strictEqual(resolveResponsePayload({}), '{}'); - assert.strictEqual(resolveResponsePayload([]), '[]'); }); it('prefers the raw response provided for paginated and long-running calls', () => { @@ -1173,7 +1173,6 @@ describe('errorUtil', () => { it('returns undefined when there is no response', () => { assert.strictEqual(resolveResponsePayload(undefined), undefined); assert.strictEqual(resolveResponsePayload(null), undefined); - assert.strictEqual(resolveResponsePayload(null, null), undefined); }); it('serializes protobuf messages through their JSON representation', () => { @@ -1187,15 +1186,6 @@ describe('errorUtil', () => { ); }); - it('handles circular references and BigInt values without throwing', () => { - const response: Record = {size: 10n}; - response.self = response; - assert.strictEqual( - resolveResponsePayload(response), - '{"size":"10","self":"[Circular]"}', - ); - }); - it('encodes binary payloads as base64', () => { assert.strictEqual( resolveResponsePayload(Buffer.from('hello')),