Skip to content

Commit 5608fb2

Browse files
aswynzclaude
andcommitted
[OPIK-8242] fix: price models with compact YYYYMMDD date suffixes
DATE_SUFFIX_PATTERN only stripped hyphen-separated dates (-2025-12-17), but Anthropic ships compact dates on every dated model id (claude-haiku-4-5-20251001). The date fallback in findModelPrice therefore never fired for them, so a compact-dated name priced correctly only when it was present verbatim in the price table. Any name needing normalization fell through every fallback to DEFAULT_COST and reported $0 -- silently, with token counts still captured. Most visible via the alias_of entries: "claude-4-6-opus" -> "claude-opus-4-6" already exists for reversed family/version ordering, so anthropic/claude-4.6-opus prices at $5/$25, but anthropic/claude-4.6-opus-20260205 resolved to $0 because the compact date could not be stripped before the alias lookup. Also hit new releases before the daily LiteLLM sync picks them up, and is not Anthropic-specific. Observed in production: 71,442 of ~131,000 LLM spans in one project priced at $0, hiding ~$4,470 of spend and inflating a 60-day cost delta to +573,216% where actual spend was flat (~+3%). Make both separators optional. Month/day ranges are kept so an arbitrary 8-digit build number is not mistaken for a date and cannot collapse a distinct model onto another model's price row. The pricing arithmetic itself was verified correct and is unchanged. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 867d65e commit 5608fb2

2 files changed

Lines changed: 80 additions & 3 deletions

File tree

apps/opik-backend/src/main/java/com/comet/opik/domain/cost/CostService.java

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,12 @@ public class CostService {
7171
public static final String MODEL_PRICES_FILE = "model_prices_and_context_window.json";
7272
public static final String MODEL_PRICES_OVERRIDES_FILE = "model_prices_overrides.json";
7373
private static final String BEDROCK_PROVIDER = "bedrock";
74-
private static final String DATE_SUFFIX_PATTERN = "-\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$";
74+
// Both hyphen-separated (2025-12-17) and compact (20251217) date suffixes. Anthropic ships
75+
// compact dates on every dated model id (claude-haiku-4-5-20251001), so without the optional
76+
// separators a compact-dated name only ever prices when it is present verbatim in the price
77+
// table -- any name needing normalization (dot form, an `alias_of` entry, or a release the
78+
// daily LiteLLM sync has not picked up yet) silently fell through to a zero cost.
79+
private static final String DATE_SUFFIX_PATTERN = "-\\d{4}-?(0[1-9]|1[0-2])-?(0[1-9]|[12]\\d|3[01])$";
7580
private static final String VERSION_SUFFIX_PATTERN = ":\\d+$";
7681
private static final Map<String, BiFunction<ModelPrice, Map<String, Integer>, BigDecimal>> PROVIDERS_CACHE_COST_CALCULATOR = Map
7782
.ofEntries(
@@ -276,7 +281,11 @@ private static String normalizeModelName(String modelName) {
276281
* This handles cases where providers return dated model names (e.g., "gpt-5.2-2025-12-17")
277282
* but the pricing database only has the base model name (e.g., "gpt-5.2").
278283
*
279-
* Date patterns recognized: YYYY-MM-DD (e.g., "2025-12-17") at the end of the model name.
284+
* Date patterns recognized at the end of the model name: YYYY-MM-DD (e.g., "2025-12-17")
285+
* and the compact YYYYMMDD form (e.g., "20251217") that Anthropic uses for every dated
286+
* model id. The separators are optional independently, so partially-separated forms are
287+
* tolerated too; month and day are still range-checked, so a plain 8-digit build number
288+
* such as "-99999999" is left alone.
280289
*
281290
* @param modelName The model name
282291
* @return Lowercase model name with date suffix removed if present, otherwise lowercase original name

apps/opik-backend/src/test/java/com/comet/opik/domain/cost/CostServiceTest.java

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import org.junit.jupiter.params.ParameterizedTest;
77
import org.junit.jupiter.params.provider.Arguments;
88
import org.junit.jupiter.params.provider.MethodSource;
9+
import org.junit.jupiter.params.provider.ValueSource;
910

1011
import java.math.BigDecimal;
1112
import java.util.Map;
@@ -375,6 +376,68 @@ void calculateCost_shouldReturnZeroForUnknownModelWithDateSuffix_issue5018() {
375376
assertThat(cost).isEqualTo(BigDecimal.ZERO);
376377
}
377378

379+
/**
380+
* A compact-dated name must price at exactly the rate of the base model it normalizes to,
381+
* not merely at some non-zero rate -- that is what distinguishes a real price-table hit from
382+
* an accidental one. Every case below was observed in production traffic routed through an
383+
* enterprise gateway, which emits Anthropic ids in reversed family/version order with a
384+
* compact date. Before the fix each of these resolved to DEFAULT_COST and reported $0.00.
385+
*/
386+
@ParameterizedTest
387+
@MethodSource("provideCompactDatedModelNamesWithBaseEquivalent")
388+
void calculateCost_compactDateSuffixPricesSameAsBaseModel(String datedModelName, String baseModelName,
389+
String provider) {
390+
Map<String, Integer> usage = Map.of(
391+
"prompt_tokens", 1000,
392+
"completion_tokens", 500);
393+
394+
BigDecimal datedCost = CostService.calculateCost(datedModelName, provider, usage, null);
395+
BigDecimal baseCost = CostService.calculateCost(baseModelName, provider, usage, null);
396+
397+
assertThat(baseCost).isGreaterThan(BigDecimal.ZERO);
398+
assertThat(datedCost).isEqualByComparingTo(baseCost);
399+
}
400+
401+
private static Stream<Arguments> provideCompactDatedModelNamesWithBaseEquivalent() {
402+
return Stream.of(
403+
// Reversed family/version order reaches the price row through an `alias_of` entry,
404+
// which is only reachable once the compact date is stripped.
405+
Arguments.of("anthropic/claude-4.6-opus-20260205", "claude-opus-4-6", "anthropic"),
406+
Arguments.of("anthropic/claude-4.6-sonnet-20260217", "claude-sonnet-4-6", "anthropic"),
407+
Arguments.of("anthropic/claude-4.5-haiku-20251001", "claude-haiku-4-5", "anthropic"),
408+
// Same path without the provider prefix.
409+
Arguments.of("claude-4.6-opus-20260205", "claude-opus-4-6", "anthropic"),
410+
// Canonical order, compact date, dot form: reaches the row via dot normalization.
411+
Arguments.of("claude-opus-4.6-20260205", "claude-opus-4-6", "anthropic"));
412+
}
413+
414+
@Test
415+
void calculateCost_shouldReturnZeroForUnknownModelWithCompactDateSuffix() {
416+
Map<String, Integer> usage = Map.of(
417+
"prompt_tokens", 1000,
418+
"completion_tokens", 500);
419+
420+
BigDecimal cost = CostService.calculateCost("unknown-model-20251217", "openai", usage, null);
421+
422+
assertThat(cost).isEqualTo(BigDecimal.ZERO);
423+
}
424+
425+
/**
426+
* The month/day ranges in the pattern keep an arbitrary 8-digit build or revision number from
427+
* being mistaken for a date and silently collapsing a distinct model onto another model's row.
428+
*/
429+
@ParameterizedTest
430+
@ValueSource(strings = {"gpt-5.2-99999999", "gpt-5.2-20251345", "gpt-5.2-12345678"})
431+
void calculateCost_shouldNotStripNonDateEightDigitSuffix(String modelName) {
432+
Map<String, Integer> usage = Map.of(
433+
"prompt_tokens", 1000,
434+
"completion_tokens", 500);
435+
436+
BigDecimal cost = CostService.calculateCost(modelName, "openai", usage, null);
437+
438+
assertThat(cost).isEqualTo(BigDecimal.ZERO);
439+
}
440+
378441
/**
379442
* Test for issue #5621: LiteLLM OTel model names with provider prefix not found in pricing table.
380443
*
@@ -965,7 +1028,12 @@ private static Stream<Arguments> provideModelNamesWithDateSuffixes() {
9651028
Arguments.of("claude-sonnet-4.5", "anthropic"),
9661029
// 4. Provider prefix + date suffix: prefix stripped first, then date suffix removed
9671030
Arguments.of("anthropic/claude-sonnet-4.5-2025-12-17", "anthropic"),
968-
Arguments.of("openai/gpt-5.2-2025-12-17", "openai"));
1031+
Arguments.of("openai/gpt-5.2-2025-12-17", "openai"),
1032+
// 5. Compact YYYYMMDD dates, the form Anthropic actually ships on every dated id.
1033+
Arguments.of("gpt-5.2-20251217", "openai"),
1034+
Arguments.of("claude-sonnet-4.5-20251217", "anthropic"),
1035+
Arguments.of("anthropic/claude-sonnet-4.5-20251217", "anthropic"),
1036+
Arguments.of("openai/gpt-5.2-20251217", "openai"));
9691037
}
9701038

9711039
/**

0 commit comments

Comments
 (0)