fix(langchain): stop emitting space-separated provider names that price at $0 (#666)

* fix(langchain): stop emitting space-separated provider names that price at $0

Cost is resolved by lowercasing the step's `provider` and matching an
llm-costs slug exactly. Matching normalizes case but NOT separators, so a
space makes the value unmatchable and the row silently prices at zero.
Verified against the live data-stream API with identical model and token
counts:

    provider="Together AI"  -> cost 0.0
    provider="Together_AI"  -> cost 0.0065
    provider="together_ai"  -> cost 0.0065

Two causes, both fixed:

1. The unmapped-`ls_provider` fallback ran `.replace("_", " ").title()`,
   converting values LangChain already supplies as valid slugs into
   unpriceable display strings ("vercel_ai_gateway" -> "Vercel Ai Gateway").
   Dropping the `.replace()` fixes every unmapped multi-token provider;
   `.title()` is retained since matching is case-insensitive.

2. Six map values used literal spaces. "Together AI" -> "Together_AI" and
   "Fireworks AI" -> "Fireworks_AI" in both LS_PROVIDER_TO_OPENLAYER_MAP and
   LITELLM_PREFIX_TO_PROVIDER_MAP.

"Hugging Face" -> "HuggingFace" for consistency only: llm-costs publishes no
huggingface provider slug at all, so that provider remains unpriced either way.

Behavior change: `step.provider` for an unmapped `ls_provider` now keeps
underscores ("Some_New_Provider" rather than "Some New Provider"). Consumers
displaying this field verbatim will see the separator.

Adds TestProviderCostSlugs, which asserts no map value contains a space and
that known prefixes resolve to real slugs, with already-correct providers
(gemini, vertex_ai, anthropic, bedrock, deepseek) kept as regression canaries.

* test(langchain): assert provider values resolve a real cost slug

The space check catches the specific bug but not the general class: a value can
be space-free and still not price anything. Port the invariant openlayer-ts
asserts through PROVIDER_COST_SLUG -- every provider value must lowercase to a
slug published by llm-costs, or be an explicitly listed display-only vendor.

The slug set is vendored so the suite stays offline. huggingface is the sole
unpriced entry: llm-costs publishes no hugging* provider at all.

Verified red/green: against pristine main this fails with
  LS_PROVIDER_TO_OPENLAYER_MAP['fireworks'] = 'Fireworks AI' lowercases to
  'fireworks ai', which is neither a published cost slug nor a known-unpriced
  vendor: every row would cost $0

Author: viniciusdsmello

src/openlayer/lib/integrations/langchain_callback.py

@@ -24,6 +24,10 @@
 from .. import utils
 from ..tracing import enums, steps, tracer, traces
 
+# Provider values below must stay matchable against the llm-costs table: cost is
+# resolved by lowercasing the provider and matching a slug exactly. Matching
+# normalizes case but NOT separators, so "Together AI" prices every row at $0
+# while "Together_AI" prices correctly. Never use spaces in these values.
 LANGCHAIN_TO_OPENLAYER_PROVIDER_MAP = {
     "azure-openai-chat": "Azure",
     "openai-chat": "OpenAI",
@@ -56,12 +60,12 @@
     "bedrock": "Bedrock",
     "amazon_bedrock": "Bedrock",
     "ollama": "Ollama",
-    "huggingface": "Hugging Face",
+    "huggingface": "HuggingFace",
     "replicate": "Replicate",
-    "together": "Together AI",
+    "together": "Together_AI",
     "groq": "Groq",
     "deepseek": "DeepSeek",
-    "fireworks": "Fireworks AI",
+    "fireworks": "Fireworks_AI",
     "perplexity": "Perplexity",
 }
 
@@ -77,12 +81,12 @@
     "bedrock": "Bedrock",
     "vertex_ai": "Google",
     "azure": "Azure",
-    "huggingface": "Hugging Face",
+    "huggingface": "HuggingFace",
     "replicate": "Replicate",
-    "together_ai": "Together AI",
+    "together_ai": "Together_AI",
     "groq": "Groq",
     "deepseek": "DeepSeek",
-    "fireworks_ai": "Fireworks AI",
+    "fireworks_ai": "Fireworks_AI",
     "perplexity": "Perplexity",
     "ollama": "Ollama",
     "openai": "OpenAI",
@@ -484,7 +488,10 @@ def _extract_model_info(
         #   3. LiteLLM model prefix (handled below)
         ls_provider = metadata.get("ls_provider")
         if ls_provider:
-            provider = LS_PROVIDER_TO_OPENLAYER_MAP.get(ls_provider, str(ls_provider).replace("_", " ").title())
+            # Keep separators: cost is matched by lowercasing this value against a
+            # llm-costs slug, and matching normalizes case but NOT separators. Turning
+            # "vercel_ai_gateway" into "Vercel Ai Gateway" makes the row price at $0.
+            provider = LS_PROVIDER_TO_OPENLAYER_MAP.get(ls_provider, str(ls_provider).title())
         else:
             provider = invocation_params.get("_type")
             if provider in LANGCHAIN_TO_OPENLAYER_PROVIDER_MAP:

tests/lib/integrations/test_langchain_callback.py

@@ -35,7 +35,12 @@
 from langchain_core.documents import Document
 
 from openlayer.lib.tracing import enums, steps
-from openlayer.lib.integrations.langchain_callback import OpenlayerHandler
+from openlayer.lib.integrations.langchain_callback import (
+    LS_PROVIDER_TO_OPENLAYER_MAP,
+    LITELLM_PREFIX_TO_PROVIDER_MAP,
+    LANGCHAIN_TO_OPENLAYER_PROVIDER_MAP,
+    OpenlayerHandler,
+)
 
 
 @pytest.fixture(autouse=True)
@@ -400,13 +405,19 @@ def test_ls_provider_is_primary_source(self) -> None:
         assert info["model"] == "claude-x"
 
     def test_ls_provider_title_cases_unknown_values(self) -> None:
+        """Unknown ls_provider values are title-cased but keep their separators.
+
+        Underscores must survive: the cost table is keyed by slugs like
+        ``vercel_ai_gateway``, and replacing "_" with " " makes the provider
+        un-priceable. See TestProviderCostSlugs.
+        """
         handler = OpenlayerHandler()
         info = handler._extract_model_info(
             serialized={},
             invocation_params={"model": "some-model"},
             metadata={"ls_provider": "some_new_provider"},
         )
-        assert info["provider"] == "Some New Provider"
+        assert info["provider"] == "Some_New_Provider"
 
     def test_falls_back_to_type_map_without_ls_provider(self) -> None:
         handler = OpenlayerHandler()
@@ -701,3 +712,127 @@ def test_usage_details_set_on_step_via_callbacks(self) -> None:
 
     def test_step_omits_usage_details_when_unset(self) -> None:
         assert "usageDetails" not in steps.ChatCompletionStep(name="x").to_dict()
+
+
+# --------------------------------------------------------------------------- #
+# Provider values must be matchable against the llm-costs table
+# --------------------------------------------------------------------------- #
+class TestProviderCostSlugs:
+    """Cost is resolved by lowercasing ``provider`` and matching a llm-costs slug.
+
+    Matching normalizes case but NOT separators (verified against the live API:
+    provider "Together AI" priced at $0.0, while "Together_AI" and "together_ai"
+    both priced at $0.0065 for the same model and token counts). So any provider
+    value containing a space silently prices every row at zero.
+    """
+
+    ALL_MAPS = (
+        ("LANGCHAIN_TO_OPENLAYER_PROVIDER_MAP", LANGCHAIN_TO_OPENLAYER_PROVIDER_MAP),
+        ("LS_PROVIDER_TO_OPENLAYER_MAP", LS_PROVIDER_TO_OPENLAYER_MAP),
+        ("LITELLM_PREFIX_TO_PROVIDER_MAP", LITELLM_PREFIX_TO_PROVIDER_MAP),
+    )
+
+    # Provider slugs published by https://llm-costs.openlayer.com/v1/costs that these
+    # maps target. Vendored rather than fetched so the suite stays offline; refresh with
+    #   curl -s https://llm-costs.openlayer.com/v1/costs | jq -r '.costs[].provider' | sort -u
+    # Last verified 2026-08-20 against 139 published providers.
+    COST_SLUGS = frozenset(
+        {
+            "anthropic",
+            "azure",
+            "bedrock",
+            "cohere",
+            "deepseek",
+            "fireworks_ai",
+            "google",
+            "groq",
+            "mistral",
+            "ollama",
+            "openai",
+            "perplexity",
+            "replicate",
+            "together_ai",
+        }
+    )
+
+    # Vendors with no provider slug upstream at all: no value can resolve a cost, so the
+    # name is display-only. Mirrors the ``null`` entries in openlayer-ts's
+    # PROVIDER_COST_SLUG.
+    UNPRICED_VENDORS = frozenset({"huggingface"})
+
+    def test_every_provider_value_resolves_a_cost_slug(self) -> None:
+        """Stronger than the space check: the value must actually price something.
+
+        Ports the invariant openlayer-ts asserts via PROVIDER_COST_SLUG -- a provider
+        is only worth mapping if its canonical name resolves a price, otherwise the
+        step gets a nicer label and still costs $0.
+        """
+        for name, mapping in self.ALL_MAPS:
+            for key, value in sorted(mapping.items()):
+                slug = value.lower()
+                assert slug in self.COST_SLUGS or slug in self.UNPRICED_VENDORS, (
+                    f"{name}[{key!r}] = {value!r} lowercases to {slug!r}, which is neither a "
+                    f"published cost slug nor a known-unpriced vendor: every row would cost $0"
+                )
+
+    def test_no_provider_value_contains_a_space(self) -> None:
+        for name, mapping in self.ALL_MAPS:
+            offenders = sorted({v for v in mapping.values() if " " in v})
+            assert not offenders, f"{name} maps to space-separated values (silent $0): {offenders}"
+
+    @pytest.mark.parametrize(
+        ("prefix", "expected_slug"),
+        [
+            # regressions this class was added for
+            ("together_ai", "together_ai"),
+            ("fireworks_ai", "fireworks_ai"),
+            ("huggingface", "huggingface"),
+            # canaries: these already resolved correctly and must keep doing so
+            ("gemini", "google"),
+            ("vertex_ai", "google"),
+            ("anthropic", "anthropic"),
+            ("bedrock", "bedrock"),
+            ("deepseek", "deepseek"),
+        ],
+    )
+    def test_litellm_prefix_resolves_to_cost_slug(self, prefix: str, expected_slug: str) -> None:
+        handler = OpenlayerHandler()
+        info = handler._extract_model_info(
+            serialized={},
+            invocation_params={"model": f"{prefix}/some-model"},
+            metadata={},
+        )
+        assert info["provider"] is not None
+        assert info["provider"].lower() == expected_slug
+        assert info["model"] == "some-model"
+
+    @pytest.mark.parametrize(
+        ("ls_provider", "expected_slug"),
+        [
+            ("together", "together_ai"),
+            ("fireworks", "fireworks_ai"),
+            ("huggingface", "huggingface"),
+            ("anthropic", "anthropic"),
+            ("google_genai", "google"),
+        ],
+    )
+    def test_ls_provider_resolves_to_cost_slug(self, ls_provider: str, expected_slug: str) -> None:
+        handler = OpenlayerHandler()
+        info = handler._extract_model_info(
+            serialized={},
+            invocation_params={"model": "some-model"},
+            metadata={"ls_provider": ls_provider},
+        )
+        assert info["provider"] is not None
+        assert info["provider"].lower() == expected_slug
+
+    def test_unmapped_ls_provider_preserves_slug_separators(self) -> None:
+        """An unmapped ls_provider that IS already a valid slug must stay matchable."""
+        handler = OpenlayerHandler()
+        info = handler._extract_model_info(
+            serialized={},
+            invocation_params={"model": "some-model"},
+            metadata={"ls_provider": "vercel_ai_gateway"},
+        )
+        assert info["provider"] is not None
+        assert info["provider"].lower() == "vercel_ai_gateway"