Skip to content

Commit 59fa9aa

Browse files
authored
feat(tracing): add export-stage OTEL span masking and media detection (#1646)
1 parent 53d98c9 commit 59fa9aa

15 files changed

Lines changed: 2299 additions & 56 deletions

‎.github/workflows/ci.yml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,11 @@ jobs:
219219
uv run --frozen python --version
220220
mapfile -t e2e_files < "$RUNNER_TEMP/e2e-shard-files.txt"
221221
set +e
222-
uv run --frozen pytest -n 4 --dist worksteal -s -v --log-cli-level=INFO "${e2e_files[@]}" -m "not serial_e2e"
222+
uv run --frozen pytest \
223+
-n 4 --dist worksteal \
224+
-s -v --log-cli-level=INFO \
225+
--timeout=180 --timeout-method=thread --durations=20 \
226+
"${e2e_files[@]}" -m "not serial_e2e"
223227
status=$?
224228
set -e
225229
if [ "$status" -eq 5 ]; then
@@ -233,7 +237,10 @@ jobs:
233237
run: |
234238
mapfile -t e2e_files < "$RUNNER_TEMP/e2e-shard-files.txt"
235239
set +e
236-
uv run --frozen pytest -s -v --log-cli-level=INFO "${e2e_files[@]}" -m "serial_e2e"
240+
uv run --frozen pytest \
241+
-s -v --log-cli-level=INFO \
242+
--timeout=180 --timeout-method=thread --durations=20 \
243+
"${e2e_files[@]}" -m "serial_e2e"
237244
status=$?
238245
set -e
239246
if [ "$status" -eq 5 ]; then
@@ -246,7 +253,11 @@ jobs:
246253
if: ${{ matrix.suite == 'live_provider' }}
247254
run: |
248255
uv run --frozen python --version
249-
uv run --frozen pytest -n 4 --dist worksteal -s -v --log-cli-level=INFO tests/live_provider -m "live_provider"
256+
uv run --frozen pytest \
257+
-n 4 --dist worksteal \
258+
-s -v --log-cli-level=INFO \
259+
--timeout=180 --timeout-method=thread --durations=20 \
260+
tests/live_provider -m "live_provider"
250261
251262
all-tests-passed:
252263
# This allows us to have a branch protection rule for tests and deploys with matrix

‎langfuse/__init__.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,14 @@
3636
is_known_llm_instrumentor,
3737
is_langfuse_span,
3838
)
39+
from .types import (
40+
MaskOtelSpansFunction,
41+
MaskOtelSpansParams,
42+
MaskOtelSpansResult,
43+
OtelSpanData,
44+
OtelSpanIdentifier,
45+
OtelSpanPatch,
46+
)
3947

4048
Langfuse = _client_module.Langfuse
4149

@@ -71,6 +79,12 @@
7179
"is_genai_span",
7280
"is_known_llm_instrumentor",
7381
"KNOWN_LLM_INSTRUMENTATION_SCOPE_PREFIXES",
82+
"MaskOtelSpansFunction",
83+
"MaskOtelSpansParams",
84+
"MaskOtelSpansResult",
85+
"OtelSpanData",
86+
"OtelSpanIdentifier",
87+
"OtelSpanPatch",
7488
"experiment",
7589
"api",
7690
]

‎langfuse/_client/client.py‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,13 @@
134134
PromptClient,
135135
TextPromptClient,
136136
)
137-
from langfuse.types import MaskFunction, ScoreDataType, SpanLevel, TraceContext
137+
from langfuse.types import (
138+
MaskFunction,
139+
MaskOtelSpansFunction,
140+
ScoreDataType,
141+
SpanLevel,
142+
TraceContext,
143+
)
138144

139145

140146
class Langfuse:
@@ -171,7 +177,40 @@ class Langfuse:
171177
release (Optional[str]): Release version/hash of your application. Used for grouping analytics by release.
172178
media_upload_thread_count (Optional[int]): Number of background threads for handling media uploads. Defaults to 1. Can also be set via LANGFUSE_MEDIA_UPLOAD_THREAD_COUNT environment variable.
173179
sample_rate (Optional[float]): Sampling rate for traces (0.0 to 1.0). Defaults to 1.0 (100% of traces are sampled). Can also be set via LANGFUSE_SAMPLE_RATE environment variable.
174-
mask (Optional[MaskFunction]): Function to mask sensitive data in traces before sending to the API.
180+
mask (Optional[MaskFunction]): Function to mask sensitive data synchronously when Langfuse SDK attributes are created. This applies only to data set through Langfuse SDK APIs such as `start_observation()`, `update()`, and `set_trace_io()`.
181+
mask_otel_spans (Optional[MaskOtelSpansFunction]): Synchronous export-stage hook for masking raw OpenTelemetry span attributes before this Langfuse client sends them to Langfuse. Use this for spans created by third-party OpenTelemetry instrumentations, or when you need to inspect final span attributes after export filtering and Langfuse media handling. It does not modify spans already exported through other OpenTelemetry exporters.
182+
183+
The hook receives one OpenTelemetry export batch. A batch is not guaranteed to contain a complete trace, request, or Langfuse observation tree. The hook usually runs on the OpenTelemetry batch span processor worker thread; during `flush()` and shutdown it may run on the caller thread. Keep it synchronous, deterministic, and fast.
184+
185+
Return `None` to leave the batch unchanged. Return `MaskOtelSpansResult` with `OtelSpanPatch` values to delete or replace attributes on selected spans. If the hook raises or returns an invalid batch result, Langfuse drops the whole export batch. If one returned span patch is invalid, Langfuse drops only that span from the Langfuse export.
186+
187+
Example:
188+
```python
189+
from typing import Optional
190+
191+
from langfuse import Langfuse
192+
from langfuse.types import (
193+
MaskOtelSpansParams,
194+
MaskOtelSpansResult,
195+
OtelSpanPatch,
196+
)
197+
198+
def mask_otel_spans(
199+
*, params: MaskOtelSpansParams
200+
) -> Optional[MaskOtelSpansResult]:
201+
patches = {}
202+
203+
for identifier, span in params.spans.items():
204+
if "gen_ai.prompt.0.content" in span.attributes:
205+
patches[identifier] = OtelSpanPatch(
206+
delete_attributes=("gen_ai.prompt.0.content",),
207+
set_attributes={"masking.applied": True},
208+
)
209+
210+
return MaskOtelSpansResult(span_patches=patches)
211+
212+
langfuse = Langfuse(mask_otel_spans=mask_otel_spans)
213+
```
175214
blocked_instrumentation_scopes (Optional[List[str]]): Deprecated. Use `should_export_span` instead. Equivalent behavior:
176215
```python
177216
from langfuse.span_filter import is_default_export_span
@@ -248,6 +287,7 @@ def __init__(
248287
media_upload_thread_count: Optional[int] = None,
249288
sample_rate: Optional[float] = None,
250289
mask: Optional[MaskFunction] = None,
290+
mask_otel_spans: Optional[MaskOtelSpansFunction] = None,
251291
blocked_instrumentation_scopes: Optional[List[str]] = None,
252292
should_export_span: Optional[Callable[[ReadableSpan], bool]] = None,
253293
additional_headers: Optional[Dict[str, str]] = None,
@@ -344,6 +384,7 @@ def __init__(
344384
media_upload_thread_count=media_upload_thread_count,
345385
sample_rate=sample_rate,
346386
mask=mask,
387+
mask_otel_spans=mask_otel_spans,
347388
tracing_enabled=self._tracing_enabled,
348389
blocked_instrumentation_scopes=blocked_instrumentation_scopes,
349390
should_export_span=should_export_span,

‎langfuse/_client/get_client.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ def _create_client_from_instance(
5050
media_upload_thread_count=instance.media_upload_thread_count,
5151
sample_rate=instance.sample_rate,
5252
mask=instance.mask,
53+
mask_otel_spans=instance.mask_otel_spans,
5354
blocked_instrumentation_scopes=instance.blocked_instrumentation_scopes,
5455
should_export_span=instance.should_export_span,
5556
additional_headers=instance.additional_headers,

‎langfuse/_client/resource_manager.py‎

Lines changed: 34 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
from langfuse._utils.request import LangfuseClient
4646
from langfuse.api import AsyncLangfuseAPI, LangfuseAPI
4747
from langfuse.logger import langfuse_logger
48-
from langfuse.types import MaskFunction
48+
from langfuse.types import MaskFunction, MaskOtelSpansFunction
4949

5050
from .._version import __version__ as langfuse_version
5151

@@ -94,6 +94,7 @@ def __new__(
9494
media_upload_thread_count: Optional[int] = None,
9595
sample_rate: Optional[float] = None,
9696
mask: Optional[MaskFunction] = None,
97+
mask_otel_spans: Optional[MaskOtelSpansFunction] = None,
9798
tracing_enabled: Optional[bool] = None,
9899
blocked_instrumentation_scopes: Optional[List[str]] = None,
99100
should_export_span: Optional[Callable[[ReadableSpan], bool]] = None,
@@ -128,6 +129,7 @@ def __new__(
128129
media_upload_thread_count=media_upload_thread_count,
129130
sample_rate=sample_rate,
130131
mask=mask,
132+
mask_otel_spans=mask_otel_spans,
131133
tracing_enabled=tracing_enabled
132134
if tracing_enabled is not None
133135
else True,
@@ -157,6 +159,7 @@ def _initialize_instance(
157159
httpx_client: Optional[httpx.Client] = None,
158160
sample_rate: Optional[float] = None,
159161
mask: Optional[MaskFunction] = None,
162+
mask_otel_spans: Optional[MaskOtelSpansFunction] = None,
160163
tracing_enabled: bool = True,
161164
blocked_instrumentation_scopes: Optional[List[str]] = None,
162165
should_export_span: Optional[Callable[[ReadableSpan], bool]] = None,
@@ -169,6 +172,7 @@ def _initialize_instance(
169172
self.tracing_enabled = tracing_enabled
170173
self.base_url = base_url
171174
self.mask = mask
175+
self.mask_otel_spans = mask_otel_spans
172176
self.environment = environment
173177

174178
# Store additional client settings for get_client() to use
@@ -184,33 +188,6 @@ def _initialize_instance(
184188
self.span_exporter = span_exporter
185189
self.tracer_provider: Optional[TracerProvider] = None
186190

187-
# OTEL Tracer
188-
if tracing_enabled:
189-
tracer_provider = tracer_provider or _init_tracer_provider(
190-
environment=environment, release=release, sample_rate=sample_rate
191-
)
192-
self.tracer_provider = tracer_provider
193-
194-
langfuse_processor = LangfuseSpanProcessor(
195-
public_key=self.public_key,
196-
secret_key=secret_key,
197-
base_url=base_url,
198-
timeout=timeout,
199-
flush_at=flush_at,
200-
flush_interval=flush_interval,
201-
blocked_instrumentation_scopes=blocked_instrumentation_scopes,
202-
should_export_span=should_export_span,
203-
additional_headers=additional_headers,
204-
span_exporter=span_exporter,
205-
)
206-
tracer_provider.add_span_processor(langfuse_processor)
207-
208-
self._otel_tracer = tracer_provider.get_tracer(
209-
LANGFUSE_TRACER_NAME,
210-
langfuse_version,
211-
attributes={"public_key": self.public_key},
212-
)
213-
214191
# API Clients
215192

216193
## API clients must be singletons because the underlying HTTPX clients
@@ -266,6 +243,35 @@ def _initialize_instance(
266243
)
267244
self._media_upload_consumers = []
268245

246+
# OTEL Tracer
247+
if tracing_enabled:
248+
tracer_provider = tracer_provider or _init_tracer_provider(
249+
environment=environment, release=release, sample_rate=sample_rate
250+
)
251+
self.tracer_provider = tracer_provider
252+
253+
langfuse_processor = LangfuseSpanProcessor(
254+
public_key=self.public_key,
255+
secret_key=secret_key,
256+
base_url=base_url,
257+
timeout=timeout,
258+
flush_at=flush_at,
259+
flush_interval=flush_interval,
260+
blocked_instrumentation_scopes=blocked_instrumentation_scopes,
261+
should_export_span=should_export_span,
262+
additional_headers=additional_headers,
263+
span_exporter=span_exporter,
264+
media_manager=self._media_manager,
265+
mask_otel_spans=mask_otel_spans,
266+
)
267+
tracer_provider.add_span_processor(langfuse_processor)
268+
269+
self._otel_tracer = tracer_provider.get_tracer(
270+
LANGFUSE_TRACER_NAME,
271+
langfuse_version,
272+
attributes={"public_key": self.public_key},
273+
)
274+
269275
media_upload_thread_count = media_upload_thread_count or max(
270276
int(os.getenv(LANGFUSE_MEDIA_UPLOAD_THREAD_COUNT, 1)), 1
271277
)

0 commit comments

Comments
 (0)