Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions core/packages/gax/src/fallbackRest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import * as serializer from 'proto3-json-serializer';
import {defaultToObjectOptions} from './fallback';
import {FetchParameters, FetchParametersMethod} from './fallbackServiceStub';
import {GoogleError} from './googleError';
import {GoogleError, setRawHttpResponsePayload} from './googleError';
import {rpcCodeFromHttpStatusCode} from './status';
import {getMatchedUrlTemplate, transcode} from './transcoding';

Expand Down Expand Up @@ -160,7 +160,7 @@ export function decodeResponse(
if (ok) {
throw err;
}
throw _undecodableErrorResponse(
const undecodableError = _undecodableErrorResponse(
rpc,
httpStatusCode,
`the response body is not JSON: ${
Expand All @@ -170,6 +170,8 @@ export function decodeResponse(
}`,
err,
);
setRawHttpResponsePayload(undecodableError, decodedString);
throw undecodableError;
}
if (!ok) {
let error: GoogleError;
Expand All @@ -179,12 +181,14 @@ export function decodeResponse(
// `parseHttpError` indexes into the body, so JSON that parses but is not
// an object — `null`, a string, a number — fails there for the same
// reason an HTML body fails above, and must not lose the status either.
throw _undecodableErrorResponse(
const undecodableError = _undecodableErrorResponse(
rpc,
httpStatusCode,
'the response body is not a JSON error object',
err,
);
setRawHttpResponsePayload(undecodableError, decodedString);
throw undecodableError;
}
// `parseHttpError` reads the status out of the response body and maps it
// onto the gRPC `code`, keeping no record of the status the transport
Expand All @@ -197,6 +201,7 @@ export function decodeResponse(
if (httpStatusCode !== undefined) {
error.httpStatusCode = httpStatusCode;
}
setRawHttpResponsePayload(error, decodedString);
throw error;
}
const message = serializer.fromProto3JSON(rpc.resolvedResponseType!, json);
Expand Down
50 changes: 48 additions & 2 deletions core/packages/gax/src/fallbackServiceStub.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import {isNodeJS} from './featureDetection';
import {StreamArrayParser} from './streamArrayParser';
import {defaultToObjectOptions} from './fallback';
import {GoogleError} from './googleError';
import {GoogleError, setRawHttpResponsePayload} from './googleError';
import {setAttemptHttpMethod} from './observability/TracerHelper';
import {rpcCodeFromHttpStatusCode, Status} from './status';
import {pipeline, PipelineSource} from 'stream';
Expand Down Expand Up @@ -234,6 +234,43 @@
return undefined;
}

const GAXIOS_REDACTED_BODY =
'<<REDACTED> - See `errorRedactor` option in `gaxios` for configuration>.';

function _isStreamLike(value: unknown): boolean {
if (!value || typeof value !== 'object') {
return false;
}
const stream = value as {pipe?: unknown; getReader?: unknown};
return (
typeof stream.pipe === 'function' || typeof stream.getReader === 'function'
);
}

function _extractRawPayloadFromRejection(
err: Error,
responseData: unknown,
httpStatus: number,
): unknown {
if (
responseData !== undefined &&
responseData !== null &&
responseData !== '' &&
responseData !== GAXIOS_REDACTED_BODY &&
!_isStreamLike(responseData)
) {
return responseData;
}
if (
err.message.length > 0 &&
err.message !== GAXIOS_REDACTED_BODY &&
err.message !== `Request failed with status code ${httpStatus}`
) {
return err.message;
}
return undefined;
}

function _toGoogleError(err: unknown, outcome: CallOutcome): unknown {
if (err instanceof GoogleError) {
return err;
Expand Down Expand Up @@ -264,13 +301,22 @@
? fetchError.status
: fetchError.response?.status;
if (typeof httpStatus === 'number') {
const responseData = fetchError.response?.data;
const parsedError =
_parseHttpErrorFromRejection(err, fetchError.response?.data) ?? error;
_parseHttpErrorFromRejection(err, responseData) ?? error;
parsedError.cause = err;
if (typeof parsedError.code !== 'number') {
parsedError.code = rpcCodeFromHttpStatusCode(httpStatus);
}
parsedError.httpStatusCode = httpStatus;
const rawPayload = _extractRawPayloadFromRejection(
err,
responseData,
httpStatus,
);
if (rawPayload !== undefined) {
setRawHttpResponsePayload(parsedError, rawPayload);
}
return parsedError;
}

Expand Down Expand Up @@ -542,7 +588,7 @@
// state, as the handlers below do.
if (err && (timedOut || !cancelRequested)) {
if (callback) {
callback(err);

Check warning on line 591 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid calling back inside of a promise
}
streamArrayParser.emit('error', err);
}
Expand All @@ -557,7 +603,7 @@
Promise.resolve(response.ok),
response.arrayBuffer(),
])
.then(([ok, buffer]: [boolean, Buffer | ArrayBuffer]) => {

Check warning on line 606 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid nesting promises
const response = responseDecoder(
rpc,
ok,
Expand All @@ -567,7 +613,7 @@
callback!(null, response);
return;
})
.catch((err: Error) => {

Check warning on line 616 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid nesting promises
// The deadline can expire after the response headers arrive but
// before the body is fully read, which rejects here rather than
// in the outer handler.
Expand All @@ -590,7 +636,7 @@
// state we recorded.
if (timedOut || !cancelRequested) {
if (callback) {
callback(callErr);

Check warning on line 639 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid calling back inside of a promise
}
streamArrayParser.emit('error', callErr);
}
Expand Down Expand Up @@ -648,12 +694,12 @@
// nobody is listening to any more.
if (timedOut || !cancelRequested) {
if (callback) {
callback(err);

Check warning on line 697 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid calling back inside of a promise
}
streamArrayParser.emit('error', err);
}
} else if (callback) {
callback(err);

Check warning on line 702 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid calling back inside of a promise
} else {
throw err;
}
Expand Down
36 changes: 36 additions & 0 deletions core/packages/gax/src/googleError.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,42 @@
const UNKNOWN_TYPE_NO_SUCH_TYPE = 'no such type';

const NUM_OF_PARTS_IN_PROTO_TYPE_NAME = 2;
const RAW_HTTP_RESPONSE_PAYLOAD = Symbol.for(
'google-gax.rawHttpResponsePayload',
);

/**
* Attaches the full raw HTTP response body non-enumerably to a GoogleError so
* telemetry debug logging can report `response.payload` without altering
* standard error formatting or serialization.
*/
export function setRawHttpResponsePayload(
err: GoogleError,
payload: unknown,
): void {
try {
Object.defineProperty(err, RAW_HTTP_RESPONSE_PAYLOAD, {
value: payload,
configurable: true,
enumerable: false,
writable: true,
});
} catch {
// Fail open if err is frozen, non-extensible, or traps defineProperty.
}
}

/**
* Reads the raw HTTP response body previously attached by
* {@link setRawHttpResponsePayload}, if any.
*/
export function getRawHttpResponsePayload(err: object): unknown {
try {
return (err as Record<symbol, unknown>)[RAW_HTTP_RESPONSE_PAYLOAD];
} catch {
return undefined;
}
}

export class GoogleError extends Error {
code?: Status;
Expand Down Expand Up @@ -205,7 +241,7 @@
};

// Return true if proto is known in protobuf.
const isDetailKnownProto = (protobuf: any, detail: any): boolean => {

Check warning on line 244 in core/packages/gax/src/googleError.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type

Check warning on line 244 in core/packages/gax/src/googleError.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
try {
const typeName = getProtoTypeNameFromFullNameType(detail['@type']);
if (typeName === ANY_PROTO_TYPE_NAME) {
Expand All @@ -222,7 +258,7 @@
};

// Check if error is unknown type encoded.
const isUnknownTypeEncodedError = (error: any): boolean => {

Check warning on line 261 in core/packages/gax/src/googleError.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
if (typeof error === 'object' && error && 'message' in error) {
return (
error.message.includes(UNKNOWN_TYPE_ENCONDED_ERROR_PREFIX) ||
Expand All @@ -235,7 +271,7 @@
// Build unknown proto as protobuf.Message<{}>.
const buildUnknownProtoAsAny = (
unknownProto: ProtobufAny,
anyProto: any,

Check warning on line 274 in core/packages/gax/src/googleError.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
): protobuf.Message<{}> => {
return anyProto.create({
type_url: unknownProto.type_url,
Expand Down
117 changes: 101 additions & 16 deletions core/packages/gax/src/observability/LoggerHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
isPreConnectionFailure,
resolveErrorInfoDomain,
resolveErrorInfoMetadataAttributes,
resolveErrorResponsePayload,
resolveErrorType,
resolveExceptionMessage,
resolveExceptionStacktrace,
Expand Down Expand Up @@ -179,6 +180,18 @@ function emitGaxLog(
}
}

/**
* Resolves a `LogSeverity` enum value. Until a `google-logging-utils` release
* containing #9718 is published to npm (this repo uses `link-workspace-packages=false`),
* the published package exports `LogSeverity` as a type only while each enum member's
* runtime value equals its uppercase level string.
*/
function resolveLogSeverity(
level: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR',
): loggingUtils.LogSeverity {
return level as loggingUtils.LogSeverity;
}
Comment thread
shivanee-p marked this conversation as resolved.
Resolved

/**
* Returns the `google-logging-utils` logger for the given GCP client service,
* defaulting to the `'google-gax'` namespace when no service name is provided.
Expand Down Expand Up @@ -216,12 +229,15 @@ export function getGaxLogger(
* @param {unknown} err - The error thrown or reported for the call.
* @param {AttemptTraceContext} dynamicArgs - Dynamic context for the RPC call or attempt.
* @param {StaticTraceContext} [staticArgs={}] - Static context for the client library.
* @param {boolean} [includeExceptionAttributes=true] - Whether to populate `exception.*`
* attributes (set on L3 client call logs, omitted on L4 transport attempt logs).
* @returns {LogPayload} The populated log payload attributes with undefined fields omitted.
*/
export function buildLogAttributes(
err: unknown,
dynamicArgs: AttemptTraceContext,
staticArgs: StaticTraceContext = {},
includeExceptionAttributes = true,
): LogPayload {
const {rawAddress, urlDomain} = resolveEndpointAndDomain(
dynamicArgs,
Expand Down Expand Up @@ -275,14 +291,16 @@ export function buildLogAttributes(
}
Object.assign(attributes, resolveErrorInfoMetadataAttributes(err));

attributes['exception.type'] = resolveExceptionType(err);
attributes['exception.message'] = resolveExceptionMessage(
err,
dynamicArgs.rpcType,
);
const stacktrace = resolveExceptionStacktrace(err);
if (stacktrace !== undefined) {
attributes['exception.stacktrace'] = stacktrace;
if (includeExceptionAttributes) {
attributes['exception.type'] = resolveExceptionType(err);
attributes['exception.message'] = resolveExceptionMessage(
err,
dynamicArgs.rpcType,
);
const stacktrace = resolveExceptionStacktrace(err);
if (stacktrace !== undefined) {
attributes['exception.stacktrace'] = stacktrace;
}
}

return attributes;
Expand Down Expand Up @@ -391,17 +409,10 @@ export function logCall<T = GaxCallResult>(
// Build structured log attributes with exception details.
const attributes = buildLogAttributes(err, dynamicArgs, staticArgs);

const warningSeverity =
(
loggingUtils as {
LogSeverity?: {WARNING: loggingUtils.LogSeverity};
}
).LogSeverity?.WARNING ?? ('WARNING' as loggingUtils.LogSeverity);

// Emit structured WARNING log.
logger(
{
severity: warningSeverity,
severity: resolveLogSeverity('WARNING'),
...(traceId ? {telemetryTraceId: traceId} : {}),
...(spanId ? {telemetrySpanId: spanId} : {}),
other: attributes,
Expand All @@ -413,3 +424,77 @@ export function logCall<T = GaxCallResult>(

return executeLoggedOperation(fn, emitL3WarnLog, isStreamCall, userCallback);
}

/**
* Executes an individual RPC transport attempt and emits a `DEBUG` log entry
* if the attempt fails across promise, callback, stream, or synchronous execution boundaries.
*
* @template T
* @param {AttemptTraceContext} attemptDynamicArgs - Dynamic context for the RPC attempt.
* @param {StaticTraceContext} staticArgs - Static context for the client library.
* @param {function} fn - The transport attempt operation to execute. Receives the wrapped
* callback when `userCallback` is supplied, otherwise `undefined`.
* @param {boolean} [isStreamCall=false] - Whether the operation returns a stream (`EventEmitter`).
* @param {APICallback} [userCallback] - Optional attempt callback.
* @returns {T} The return value of `fn`.
*/
export function logNetworkAttempt<T = GaxCallResult>(
attemptDynamicArgs: AttemptTraceContext,
staticArgs: StaticTraceContext,
fn: (loggedCallback?: APICallback) => T,
isStreamCall = false,
userCallback?: APICallback,
): T {
// Snapshot attempt context in case async rejection loses active context.
let initialContext: ReturnType<typeof context.active> | undefined;
try {
initialContext = context.active();
} catch {
// Fail open if active context retrieval throws.
}

const emitL4DebugLog = (err: unknown) => {
Comment on lines +448 to +456

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To prevent duplicate DEBUG logs for a single network attempt (for example, when both a callback and a stream error event are triggered), we should introduce a latch flag similar to hasLoggedL3 in logCall.

Suggested change
// Snapshot attempt context in case async rejection loses active context.
const initialContext = context.active();
const emitL4DebugLog = (err: unknown) => {
// Latch to emit at most one debug log per network attempt.
let hasLoggedAttempt = false;
// Snapshot attempt context in case async rejection loses active context.
const initialContext = context.active();
const emitL4DebugLog = (err: unknown) => {
// Suppress duplicate emissions for the same attempt.
if (hasLoggedAttempt) {
return;
}
hasLoggedAttempt = true;

// Resolve logger and active T4 attempt span (or outer OpenTelemetry span when GAX tracing is disabled).
const logger = getGaxLogger(staticArgs?.gcpClientService);
const activeTrace = getActiveTraceContext();
const initialTrace = initialContext
? getActiveTraceContext(initialContext)
: undefined;
const attemptSpan =
activeTrace?.attemptSpan ??
initialTrace?.attemptSpan ??
activeTrace?.span ??
initialTrace?.span;
const spanContext = attemptSpan?.spanContext();
const traceId = spanContext?.traceId;
const spanId = spanContext?.spanId;
Comment thread
shivanee-p marked this conversation as resolved.
Resolved

// Build structured log attributes without exception details and attach raw attempt response payload when present.
const attributes = buildLogAttributes(
err,
attemptDynamicArgs,
staticArgs,
false,
);
const responsePayload = resolveErrorResponsePayload(err);
if (responsePayload !== undefined) {
attributes['response.payload'] = responsePayload;
}

// Emit structured DEBUG log for the failed attempt.
const message = resolveExceptionMessage(err, attemptDynamicArgs.rpcType);
logger(
{
severity: resolveLogSeverity('DEBUG'),
...(traceId ? {telemetryTraceId: traceId} : {}),
...(spanId ? {telemetrySpanId: spanId} : {}),
other: attributes,
},
'RPC attempt failed: %s',
message || err,
);
};

// Intercept and dispatch execution boundaries for the attempt.
return executeLoggedOperation(fn, emitL4DebugLog, isStreamCall, userCallback);
}
Loading
Loading