fix(app): count legacy STATUS_CODE_ERROR spans as errors (#3181)

## Summary

The service dashboard and the LLM expressions each spelled out their own "is this an error" predicate as `lower(<status>) = 'error'`. Both are fed `source.statusCodeExpression` for trace sources (`serviceDashboard.ts:164`, `llm/lib/expressions.ts:443`), so a collector that writes the long OTel form, `STATUS_CODE_ERROR`, fell outside the predicate and those spans went uncounted. Error rate and error counts on the Services and LLM dashboards undercounted as a result.

This pulls the definition into one exported helper, `errorStatusPredicateSql` in `source.ts`, and matches both the short `Error` form and the long `STATUS_CODE_ERROR` one.

The helper is span-status only. Log severity text is a different vocabulary: the long status value can never appear in it, and the values severity text does use for errors (`fatal`, `crit` and friends) are a different set again. So the log call site (`expressions.ts:487`) keeps a predicate of its own, unchanged from before this branch, rather than borrowing the span-status one.

A single definition for span status also means the next place that needs it does not add another spelling.

## Test plan

- Updated the existing expectations in `expressions.test.ts` and `serviceDashboard.test.ts` to the widened predicate; both suites pass. Both of those assertions cover trace sources, which is the path that changes.
- Full app suite green (246 files, 4035 tests), plus `tsc --noEmit` and `eslint` with no new warnings.

No UI or layout change: this only widens which spans an existing query counts as errors. [ui-check: allow]

## Notes

- Adds a `patch` changeset.
- Two other places still carry their own error-status spelling: the service map (`useServiceMap.tsx:178`, `ServiceMapTooltip.tsx:81`) and the client-side check in `DBTraceWaterfallChart.tsx`. Both are outside this PR's files, and the waterfall one needs a JS-side predicate rather than SQL, so they are tracked in #3203 instead of widening this change.

Author: alex-fedotyev

.changeset/error-span-predicate.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/app': patch
+---
+
+Count spans recorded with the legacy `STATUS_CODE_ERROR` status as errors on the Services and LLM dashboards. The errored-span definition now lives in one place and matches both that value and the short `Error` form, so those dashboards no longer undercount errors from collectors that emit the long form.

packages/app/src/__tests__/serviceDashboard.test.ts

@@ -460,7 +460,7 @@ describe('Service Dashboard', () => {
 
       expect(result.current.isLoading).toBe(false);
       expect(result.current.expressions?.isError).toBe(
-        "lower(StatusCode) = 'error'",
+        "lower(StatusCode) IN ('error', 'status_code_error')",
       );
       expect(result.current.expressions?.isSpanKindServer).toContain(
         'SpanKind IN',

packages/app/src/llm/__tests__/expressions.test.ts

@@ -32,7 +32,9 @@ describe('getLLMExpressions', () => {
     expect(expressions.isLLMSpan).toContain(
       "mapContains(SpanAttributes, 'gen_ai.operation.name')",
     );
-    expect(expressions.isError).toBe("lower(StatusCode) = 'error'");
+    expect(expressions.isError).toBe(
+      "lower(StatusCode) IN ('error', 'status_code_error')",
+    );
     expect(expressions.durationInMillis).toBe('Duration/1e6');
     // Provided cost takes precedence in the cost expression.
     expect(expressions.costUsd.startsWith('if((greatest(')).toBe(true);
@@ -274,7 +276,9 @@ describe('getLLMExpressions', () => {
       [],
     );
     expect(expressions.model).toContain("Attrs['gen_ai.response.model']");
-    expect(expressions.isError).toBe("lower(Status) = 'error'");
+    expect(expressions.isError).toBe(
+      "lower(Status) IN ('error', 'status_code_error')",
+    );
     expect(expressions.durationInMillis).toBe('DurationNs/1e6');
   });
 });

packages/app/src/llm/lib/expressions.ts

@@ -1,5 +1,7 @@
 import { TLogSource, TTraceSource } from '@hyperdx/common-utils/dist/types';
 
+import { errorStatusPredicateSql } from '@/source';
+
 import { generateCostSqlExpression } from './cost';
 import { buildAnyKeyExistsSql, buildLLMSpanSqlPredicate } from './detect';
 
@@ -452,7 +454,7 @@ export function getLLMExpressions(source: TTraceSource, jsonColumns: string[]) {
     ...attributeExpressions,
     ...fieldExpressions,
     ...auxExpressions,
-    isError: `lower(${fieldExpressions.severityText}) = 'error'`,
+    isError: errorStatusPredicateSql(fieldExpressions.severityText),
     hasTokens: `${attributeExpressions.totalTokens} > 0`,
   };
 }
@@ -481,7 +483,13 @@ export function getLLMLogExpressions(
     ...attributeExpressions,
     service: source.serviceNameExpression || 'ServiceName',
     severityText,
-    /** Error log events (mirrors the serviceDashboard convention). */
+    /**
+     * Error log events. Severity text has its own vocabulary, so this keeps a
+     * predicate of its own rather than reusing the span-status one: the extra
+     * status value that predicate matches cannot appear in a severity-text
+     * column, and the values severity text does use for errors are a
+     * different set.
+     */
     isError: `lower(${severityText}) = 'error'`,
     /**
      * Log events that belong to LLM activity: explicit LLM markers or any

packages/app/src/serviceDashboard.ts

@@ -5,6 +5,7 @@ import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils';
 import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types';
 
 import { useColumns, useJsonColumns } from './hooks/useMetadata';
+import { errorStatusPredicateSql } from './source';
 
 const COALESCE_FIELDS_LIMIT = 100;
 
@@ -225,7 +226,7 @@ export function getExpressions(
 
   const filterExpressions = {
     isEndpointNonEmpty: `NOT empty(${auxExpressions.endpoint})`,
-    isError: `lower(${fieldExpressions.severityText}) = 'error'`,
+    isError: errorStatusPredicateSql(fieldExpressions.severityText),
     isSpanKindServer: `${fieldExpressions.spanKind} IN ('Server', 'SPAN_KIND_SERVER')`,
     isDbSpan: `${fieldExpressions.dbStatement} <> ''`,
   };

packages/app/src/source.ts

@@ -529,6 +529,20 @@ export function getDurationSecondsExpression(source: TTraceSource) {
   return `(${source.durationExpression})/1e${source.durationPrecision ?? 9}`;
 }
 
+/**
+ * The single definition of an errored span, as a SQL predicate over a span
+ * status-code expression. Matches both the short OTel status ('Error'/'ERROR')
+ * and the legacy collector form ('STATUS_CODE_ERROR'), which some collectors
+ * write instead.
+ *
+ * Span status only. Log severity text is a different vocabulary (its error
+ * values are 'fatal', 'crit' and friends, never 'status_code_error'), so log
+ * callers keep their own predicate rather than reusing this one.
+ */
+export function errorStatusPredicateSql(statusCodeExpression: string): string {
+  return `lower(${statusCodeExpression}) IN ('error', 'status_code_error')`;
+}
+
 // Aggregate functions whose output preserves the unit of the input value.
 // count and count_distinct produce dimensionless counts and should not
 // inherit the duration format.