From 81a799ec2eab773fc3b45a844e0e7e5311449d9e Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:03:10 +0000 Subject: [PATCH 01/28] fix(langfuse_otel): emit request metadata under langfuse.observation.metadata and langfuse.trace.metadata keys Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/langfuse/langfuse_otel.py | 33 +++ litellm/types/integrations/langfuse_otel.py | 1 + tests/integration/contracts.json | 6 + .../test_langfuse_otel_metadata.py | 189 ++++++++++++++++++ .../integrations/test_langfuse_otel.py | 85 ++++++++ 5 files changed, 314 insertions(+) create mode 100644 tests/integration/observability/test_langfuse_otel_metadata.py diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index a96fac32c2a..bc47070f40f 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -2,6 +2,7 @@ import base64 import json import os from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Optional from litellm._logging import verbose_logger @@ -29,6 +30,16 @@ LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/ot LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version" LANGFUSE_INGESTION_VERSION: Final = "4" +_TRACE_IDENTITY_FIELDS: Final = MappingProxyType( + { + "user_api_key_alias": "key_alias", + "user_api_key_user_id": "user_id", + "user_api_key_end_user_id": "end_user_id", + "user_api_key_team_id": "team_id", + "user_api_key_team_alias": "team_alias", + } +) + class LangfuseOtelLogger(OpenTelemetry): def __init__(self, config=None, *args, **kwargs): @@ -125,6 +136,27 @@ class LangfuseOtelLogger(OpenTelemetry): value = str(value) safe_set_attribute(span, enum_attr.value, value) + @staticmethod + def _set_request_metadata_attributes(span: Span, kwargs: dict) -> None: + from litellm.integrations.arize._utils import safe_set_attribute + from litellm.integrations.langfuse.langfuse import log_requester_metadata + from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + + standard_logging_object: Final = kwargs.get("standard_logging_object") + request_metadata: Final = ( + standard_logging_object.get("metadata") if isinstance(standard_logging_object, dict) else None + ) + if not isinstance(request_metadata, dict): + return + observation_metadata: Final = log_requester_metadata(redact_user_api_key_info(metadata=request_metadata)) + safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_METADATA.value, safe_dumps(observation_metadata)) + trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value + for source_key, target_key in _TRACE_IDENTITY_FIELDS.items(): + value = observation_metadata.get(source_key) + if value is not None: + safe_set_attribute(span, f"{trace_prefix}.{target_key}", value) + @staticmethod def _set_observation_output(span: Span, response_obj): """Helper to set observation output attributes.""" @@ -244,6 +276,7 @@ class LangfuseOtelLogger(OpenTelemetry): metadata: Final = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata) + LangfuseOtelLogger._set_request_metadata_attributes(span=span, kwargs=kwargs) messages: Final = kwargs.get("messages") if messages: diff --git a/litellm/types/integrations/langfuse_otel.py b/litellm/types/integrations/langfuse_otel.py index c58dc567cda..11f30a2bfa7 100644 --- a/litellm/types/integrations/langfuse_otel.py +++ b/litellm/types/integrations/langfuse_otel.py @@ -29,6 +29,7 @@ class LangfuseSpanAttributes(str, Enum): # ---- Observation input/output ---- OBSERVATION_INPUT = "langfuse.observation.input" OBSERVATION_OUTPUT = "langfuse.observation.output" + OBSERVATION_METADATA = "langfuse.observation.metadata" # ---- Trace-level metadata ---- TRACE_USER_ID = "user.id" diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index a8d9cf1df8b..b9c8245f51f 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -257,6 +257,12 @@ "tests/integration/observability/test_otel_text_completion_choices.py::test_otel_weave_output_keeps_text_completion_provider_fields_beside_the_synthesized_message": [ "other.observability.otel.text_completion_choices_keep_provider_fields" ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys": [ + "other.observability.langfuse_otel.request_metadata_on_langfuse_keys" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse": [ + "other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla" + ], "tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": [ "other.observability.guardrails.rewrite_reaches_correct_anthropic_positions" ], diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py new file mode 100644 index 00000000000..8f64f66508c --- /dev/null +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -0,0 +1,189 @@ +import json +import uuid +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, eventually +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + + +def _span_attributes(body: bytes) -> tuple[dict[str, object], ...]: + request: Final = ExportTraceServiceRequest.FromString(body) + return tuple( + {attribute.key: getattr(attribute.value, attribute.value.WhichOneof("value")) for attribute in span.attributes} + for resource in request.resource_spans + for scope in resource.scope_spans + for span in scope.spans + ) + + +@contextmanager +def _langfuse_rig( + gateway: Gateway, + tmp_path: Path, + settings: Mapping[str, object], + marker: str, +) -> Iterator[tuple[Gateway, Wire, Wire]]: + def upstream(request: Request) -> Reply: + assert request.target.endswith("/chat/completions"), request.target + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{marker}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "langfuse-echo"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + ).encode() + ) + + def sink(_request: Request) -> Reply: + return Reply() + + with wire_server(upstream) as provider, wire_server(sink) as collector: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"].update({"callbacks": ["langfuse_otel"], **settings}) + path: Final = tmp_path / "langfuse_otel.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy( + gateway, + tmp_path, + { + "LANGFUSE_PUBLIC_KEY": "pk-lf-integration", + "LANGFUSE_SECRET_KEY": "sk-lf-integration", + "LANGFUSE_HOST": collector.url, + }, + config=path, + workers=2, + ) as candidate: + yield candidate, provider, collector + + +def _observation_span(collector: Wire, response_id: str) -> dict[str, object]: + batches: Final = [] + + def spans() -> tuple[dict[str, object], ...]: + batches.extend(collector.drain()) + return tuple( + attributes + for batch in batches + for attributes in _span_attributes(batch.body) + if attributes.get("llm.response.id") == response_id + ) + + observed: Final = eventually(spans, lambda values: len(values) == 1, seconds=25) + return observed[0] + + +@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_on_langfuse_keys") +def test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-meta-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as (candidate, provider, collector), + candidate.scenario() as scenario, + ): + team_alias: Final = "lf-team-" + uuid.uuid4().hex + team_id: Final = scenario.team(team_alias=team_alias) + key_alias: Final = "lf-key-" + uuid.uuid4().hex + key: Final = scenario.key( + team_id=team_id, + key_alias=key_alias, + metadata={"spend_logs_metadata": {"ticket": "LIT-8283"}}, + ) + model: Final = scenario.model(api_base=provider.url + "/v1") + end_user: Final = "lf-end-user-" + uuid.uuid4().hex + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": marker}], + "user": end_user, + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["message"]["content"] == "langfuse-echo" + + attrs: Final = _observation_span(collector, response.json()["id"]) + assert "metadata" in attrs, sorted(attrs) + assert json.loads(str(attrs["metadata"]))["user_api_key_alias"] == key_alias + assert "langfuse.observation.metadata" in attrs, sorted(attrs) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + expected: Final = { + "user_api_key_alias": key_alias, + "user_api_key_team_id": team_id, + "user_api_key_team_alias": team_alias, + "user_api_key_end_user_id": end_user, + } + assert {key_: observation.get(key_) for key_ in expected} == expected, observation + assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation + assert { + attribute: attrs.get(attribute) + for attribute in ( + "langfuse.trace.metadata.key_alias", + "langfuse.trace.metadata.team_id", + "langfuse.trace.metadata.team_alias", + "langfuse.trace.metadata.end_user_id", + ) + } == { + "langfuse.trace.metadata.key_alias": key_alias, + "langfuse.trace.metadata.team_id": team_id, + "langfuse.trace.metadata.team_alias": team_alias, + "langfuse.trace.metadata.end_user_id": end_user, + }, attrs + + +@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla") +def test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-redact-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {"redact_user_api_key_info": True}, marker) as ( + candidate, + provider, + collector, + ), + candidate.scenario() as scenario, + ): + key: Final = scenario.key( + key_alias="lf-redact-" + uuid.uuid4().hex, + metadata={"spend_logs_metadata": {"ticket": "LIT-8283"}}, + ) + model: Final = scenario.model(api_base=provider.url + "/v1") + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": marker}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + + attrs: Final = _observation_span(collector, response.json()["id"]) + assert "langfuse.observation.metadata" in attrs, sorted(attrs) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + assert not [name for name in observation if name.startswith("user_api_key")], observation + assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation + assert not [name for name in attrs if name.startswith("langfuse.trace.metadata.")], sorted(attrs) diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 0a9ce55fe16..0eb2dbdbc87 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch import pytest +import litellm from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm.types.llms.openai import ResponsesAPIResponse @@ -482,6 +483,90 @@ class TestLangfuseOtelIntegration: assert isinstance(config, OpenTelemetryConfig) # Endpoint assertion removed as side effect is gone + def test_request_metadata_is_emitted_under_langfuse_observation_and_trace_metadata_keys(self, monkeypatch): + monkeypatch.setattr(litellm, "redact_user_api_key_info", False) + request_metadata = { + "user_api_key_hash": "hash123", + "user_api_key_alias": "prod-key", + "user_api_key_user_id": "user-1", + "user_api_key_end_user_id": "end-user-1", + "user_api_key_team_id": "team-1", + "user_api_key_team_alias": "team-a", + "spend_logs_metadata": {"env": "prod"}, + "requester_ip_address": "10.0.0.1", + "requester_metadata": {"ticket": "LIT-8283"}, + } + kwargs = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": {"metadata": request_metadata}, + } + + with patch( + "litellm.integrations.arize._utils.safe_set_attribute" + ) as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + + actual = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + + assert json.loads(actual["langfuse.observation.metadata"]) == {**request_metadata} + assert { + key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.") + } == { + "langfuse.trace.metadata.key_alias": "prod-key", + "langfuse.trace.metadata.user_id": "user-1", + "langfuse.trace.metadata.end_user_id": "end-user-1", + "langfuse.trace.metadata.team_id": "team-1", + "langfuse.trace.metadata.team_alias": "team-a", + } + + def test_request_metadata_redaction_matches_vanilla_langfuse(self, monkeypatch): + monkeypatch.setattr(litellm, "redact_user_api_key_info", True) + request_metadata = { + "user_api_key_hash": "hash123", + "user_api_key_alias": "prod-key", + "user_api_key_user_id": "user-1", + "user_api_key_end_user_id": "end-user-1", + "user_api_key_team_id": "team-1", + "user_api_key_team_alias": "team-a", + "spend_logs_metadata": {"env": "prod"}, + "requester_ip_address": "10.0.0.1", + "requester_metadata": {"ticket": "LIT-8283"}, + } + kwargs = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": {"metadata": request_metadata}, + } + + with patch( + "litellm.integrations.arize._utils.safe_set_attribute" + ) as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + + actual = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + + assert json.loads(actual["langfuse.observation.metadata"]) == { + "spend_logs_metadata": {"env": "prod"}, + "requester_ip_address": "10.0.0.1", + "requester_metadata": {"ticket": "LIT-8283"}, + } + assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] + + def test_request_metadata_keys_are_absent_without_standard_logging_object(self, monkeypatch): + monkeypatch.setattr(litellm, "redact_user_api_key_info", False) + kwargs = {"litellm_params": {"metadata": {"trace_metadata": {"k": "v"}}}} + + with patch( + "litellm.integrations.arize._utils.safe_set_attribute" + ) as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + + actual = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + + assert [key for key in actual if key.startswith("langfuse.trace.metadata")] == [ + "langfuse.trace.metadata" + ] + assert "langfuse.observation.metadata" not in actual + class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target From 1201677f178ab82741e5558c11499b38ac2ec5e3 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:23:52 +0000 Subject: [PATCH 02/28] fix(langfuse_otel): name trace metadata identity keys after their request metadata fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/langfuse/langfuse_otel.py | 21 ++++++++----------- .../test_langfuse_otel_metadata.py | 18 +++++++++------- .../integrations/test_langfuse_otel.py | 10 ++++----- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index bc47070f40f..a2cd3b2221a 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -2,7 +2,6 @@ import base64 import json import os from datetime import datetime -from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Optional from litellm._logging import verbose_logger @@ -30,14 +29,12 @@ LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/ot LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version" LANGFUSE_INGESTION_VERSION: Final = "4" -_TRACE_IDENTITY_FIELDS: Final = MappingProxyType( - { - "user_api_key_alias": "key_alias", - "user_api_key_user_id": "user_id", - "user_api_key_end_user_id": "end_user_id", - "user_api_key_team_id": "team_id", - "user_api_key_team_alias": "team_alias", - } +_TRACE_IDENTITY_FIELDS: Final = ( + "user_api_key_alias", + "user_api_key_user_id", + "user_api_key_end_user_id", + "user_api_key_team_id", + "user_api_key_team_alias", ) @@ -152,10 +149,10 @@ class LangfuseOtelLogger(OpenTelemetry): observation_metadata: Final = log_requester_metadata(redact_user_api_key_info(metadata=request_metadata)) safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_METADATA.value, safe_dumps(observation_metadata)) trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value - for source_key, target_key in _TRACE_IDENTITY_FIELDS.items(): - value = observation_metadata.get(source_key) + for field in _TRACE_IDENTITY_FIELDS: + value = observation_metadata.get(field) if value is not None: - safe_set_attribute(span, f"{trace_prefix}.{target_key}", value) + safe_set_attribute(span, f"{trace_prefix}.{field}", value) @staticmethod def _set_observation_output(span: Span, response_obj): diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 8f64f66508c..03b828b0c50 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -31,6 +31,8 @@ def _langfuse_rig( marker: str, ) -> Iterator[tuple[Gateway, Wire, Wire]]: def upstream(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"data":[]}') assert request.target.endswith("/chat/completions"), request.target return Reply( body=json.dumps( @@ -138,16 +140,16 @@ def test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_tra assert { attribute: attrs.get(attribute) for attribute in ( - "langfuse.trace.metadata.key_alias", - "langfuse.trace.metadata.team_id", - "langfuse.trace.metadata.team_alias", - "langfuse.trace.metadata.end_user_id", + "langfuse.trace.metadata.user_api_key_alias", + "langfuse.trace.metadata.user_api_key_team_id", + "langfuse.trace.metadata.user_api_key_team_alias", + "langfuse.trace.metadata.user_api_key_end_user_id", ) } == { - "langfuse.trace.metadata.key_alias": key_alias, - "langfuse.trace.metadata.team_id": team_id, - "langfuse.trace.metadata.team_alias": team_alias, - "langfuse.trace.metadata.end_user_id": end_user, + "langfuse.trace.metadata.user_api_key_alias": key_alias, + "langfuse.trace.metadata.user_api_key_team_id": team_id, + "langfuse.trace.metadata.user_api_key_team_alias": team_alias, + "langfuse.trace.metadata.user_api_key_end_user_id": end_user, }, attrs diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 0eb2dbdbc87..9fed4ef0d69 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -512,11 +512,11 @@ class TestLangfuseOtelIntegration: assert { key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.") } == { - "langfuse.trace.metadata.key_alias": "prod-key", - "langfuse.trace.metadata.user_id": "user-1", - "langfuse.trace.metadata.end_user_id": "end-user-1", - "langfuse.trace.metadata.team_id": "team-1", - "langfuse.trace.metadata.team_alias": "team-a", + "langfuse.trace.metadata.user_api_key_alias": "prod-key", + "langfuse.trace.metadata.user_api_key_user_id": "user-1", + "langfuse.trace.metadata.user_api_key_end_user_id": "end-user-1", + "langfuse.trace.metadata.user_api_key_team_id": "team-1", + "langfuse.trace.metadata.user_api_key_team_alias": "team-a", } def test_request_metadata_redaction_matches_vanilla_langfuse(self, monkeypatch): From 732d2c58e60221040d49fbe040aac9282831993f Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:43:48 +0000 Subject: [PATCH 03/28] fix(langfuse_otel): type metadata helper and drop null fields so they cannot clobber caller trace metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/langfuse/langfuse_otel.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index a2cd3b2221a..193cab0cb9f 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -134,7 +134,7 @@ class LangfuseOtelLogger(OpenTelemetry): safe_set_attribute(span, enum_attr.value, value) @staticmethod - def _set_request_metadata_attributes(span: Span, kwargs: dict) -> None: + def _set_request_metadata_attributes(span: Span, kwargs: dict[str, object]) -> None: from litellm.integrations.arize._utils import safe_set_attribute from litellm.integrations.langfuse.langfuse import log_requester_metadata from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info @@ -146,12 +146,15 @@ class LangfuseOtelLogger(OpenTelemetry): ) if not isinstance(request_metadata, dict): return - observation_metadata: Final = log_requester_metadata(redact_user_api_key_info(metadata=request_metadata)) + observation_metadata: Final = { + key: value + for key, value in log_requester_metadata(redact_user_api_key_info(metadata=request_metadata)).items() + if value is not None + } safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_METADATA.value, safe_dumps(observation_metadata)) trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value for field in _TRACE_IDENTITY_FIELDS: - value = observation_metadata.get(field) - if value is not None: + if (value := observation_metadata.get(field)) is not None: safe_set_attribute(span, f"{trace_prefix}.{field}", value) @staticmethod From 4ab39e1100bb27b4b46e965057dd51fed780ba4c Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:54:18 +0000 Subject: [PATCH 04/28] test(langfuse_otel): annotate new request metadata tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/test_langfuse_otel.py | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 9fed4ef0d69..6a55a1b3106 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -1,5 +1,6 @@ import json import os +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -483,9 +484,11 @@ class TestLangfuseOtelIntegration: assert isinstance(config, OpenTelemetryConfig) # Endpoint assertion removed as side effect is gone - def test_request_metadata_is_emitted_under_langfuse_observation_and_trace_metadata_keys(self, monkeypatch): + def test_request_metadata_is_emitted_under_langfuse_observation_and_trace_metadata_keys( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setattr(litellm, "redact_user_api_key_info", False) - request_metadata = { + request_metadata: Final = { "user_api_key_hash": "hash123", "user_api_key_alias": "prod-key", "user_api_key_user_id": "user-1", @@ -496,22 +499,18 @@ class TestLangfuseOtelIntegration: "requester_ip_address": "10.0.0.1", "requester_metadata": {"ticket": "LIT-8283"}, } - kwargs = { + kwargs: Final = { "litellm_params": {"metadata": {}}, "standard_logging_object": {"metadata": request_metadata}, } - with patch( - "litellm.integrations.arize._utils.safe_set_attribute" - ) as mock_safe_set_attribute: + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) - actual = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} assert json.loads(actual["langfuse.observation.metadata"]) == {**request_metadata} - assert { - key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.") - } == { + assert {key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.")} == { "langfuse.trace.metadata.user_api_key_alias": "prod-key", "langfuse.trace.metadata.user_api_key_user_id": "user-1", "langfuse.trace.metadata.user_api_key_end_user_id": "end-user-1", @@ -519,9 +518,9 @@ class TestLangfuseOtelIntegration: "langfuse.trace.metadata.user_api_key_team_alias": "team-a", } - def test_request_metadata_redaction_matches_vanilla_langfuse(self, monkeypatch): + def test_request_metadata_redaction_matches_vanilla_langfuse(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "redact_user_api_key_info", True) - request_metadata = { + request_metadata: Final = { "user_api_key_hash": "hash123", "user_api_key_alias": "prod-key", "user_api_key_user_id": "user-1", @@ -532,17 +531,15 @@ class TestLangfuseOtelIntegration: "requester_ip_address": "10.0.0.1", "requester_metadata": {"ticket": "LIT-8283"}, } - kwargs = { + kwargs: Final = { "litellm_params": {"metadata": {}}, "standard_logging_object": {"metadata": request_metadata}, } - with patch( - "litellm.integrations.arize._utils.safe_set_attribute" - ) as mock_safe_set_attribute: + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) - actual = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} assert json.loads(actual["langfuse.observation.metadata"]) == { "spend_logs_metadata": {"env": "prod"}, @@ -551,20 +548,18 @@ class TestLangfuseOtelIntegration: } assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] - def test_request_metadata_keys_are_absent_without_standard_logging_object(self, monkeypatch): + def test_request_metadata_keys_are_absent_without_standard_logging_object( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setattr(litellm, "redact_user_api_key_info", False) - kwargs = {"litellm_params": {"metadata": {"trace_metadata": {"k": "v"}}}} + kwargs: Final = {"litellm_params": {"metadata": {"trace_metadata": {"k": "v"}}}} - with patch( - "litellm.integrations.arize._utils.safe_set_attribute" - ) as mock_safe_set_attribute: + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) - actual = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - assert [key for key in actual if key.startswith("langfuse.trace.metadata")] == [ - "langfuse.trace.metadata" - ] + assert [key for key in actual if key.startswith("langfuse.trace.metadata")] == ["langfuse.trace.metadata"] assert "langfuse.observation.metadata" not in actual From bfa32f66167f1fbe952f1204f6fa214fe04f78d1 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 08:06:28 +0000 Subject: [PATCH 05/28] test(langfuse_otel): build rig config and wire batches immutably Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../observability/test_langfuse_otel_metadata.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 03b828b0c50..fb07b672f7e 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -57,8 +57,11 @@ def _langfuse_rig( return Reply() with wire_server(upstream) as provider, wire_server(sink) as collector: - config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) - config["litellm_settings"].update({"callbacks": ["langfuse_otel"], **settings}) + raw_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config: Final = { + **raw_config, + "litellm_settings": {**raw_config["litellm_settings"], "callbacks": ["langfuse_otel"], **settings}, + } path: Final = tmp_path / "langfuse_otel.yaml" path.write_text(yaml.safe_dump(config)) with owned_proxy( @@ -76,7 +79,7 @@ def _langfuse_rig( def _observation_span(collector: Wire, response_id: str) -> dict[str, object]: - batches: Final = [] + batches: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, so accumulating needs a mutable list def spans() -> tuple[dict[str, object], ...]: batches.extend(collector.drain()) From 7328bb86af83399cb5205ce287f716d22ac95c2a Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 08:21:47 +0000 Subject: [PATCH 06/28] fix(langfuse_otel): skip empty trace identity values so teamless requests cannot erase attribution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/langfuse/langfuse_otel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 193cab0cb9f..e7e4b16e866 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -154,7 +154,7 @@ class LangfuseOtelLogger(OpenTelemetry): safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_METADATA.value, safe_dumps(observation_metadata)) trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value for field in _TRACE_IDENTITY_FIELDS: - if (value := observation_metadata.get(field)) is not None: + if value := observation_metadata.get(field): safe_set_attribute(span, f"{trace_prefix}.{field}", value) @staticmethod From bc6d6a516da8155de9306b3fc5408dd14c887904 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 08:33:52 +0000 Subject: [PATCH 07/28] test(langfuse_otel): drain wire batches per poll instead of accumulating Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integration/observability/test_langfuse_otel_metadata.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index fb07b672f7e..bc22f187528 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -79,13 +79,10 @@ def _langfuse_rig( def _observation_span(collector: Wire, response_id: str) -> dict[str, object]: - batches: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, so accumulating needs a mutable list - def spans() -> tuple[dict[str, object], ...]: - batches.extend(collector.drain()) return tuple( attributes - for batch in batches + for batch in collector.drain() for attributes in _span_attributes(batch.body) if attributes.get("llm.response.id") == response_id ) From 2eead5ee026e7df9fbca4da840c666edbf3d6f35 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 08:45:38 +0000 Subject: [PATCH 08/28] test(langfuse_otel): cover null stripping and empty trace identities Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/test_langfuse_otel.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 6a55a1b3106..720b81712f4 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -548,6 +548,42 @@ class TestLangfuseOtelIntegration: } assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] + def test_request_metadata_drops_null_fields_and_skips_empty_trace_identities( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "redact_user_api_key_info", False) + request_metadata: Final = { + "user_api_key_alias": "prod-key", + "user_api_key_user_id": "user-1", + "user_api_key_end_user_id": None, + "user_api_key_team_id": "", + "user_api_key_team_alias": None, + "team_id": None, + "team_alias": None, + "spend_logs_metadata": {"env": "prod"}, + } + kwargs: Final = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": {"metadata": request_metadata}, + } + + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + + assert json.loads(actual["langfuse.observation.metadata"]) == { + "user_api_key_alias": "prod-key", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "", + "spend_logs_metadata": {"env": "prod"}, + "requester_metadata": {}, + } + assert {key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.")} == { + "langfuse.trace.metadata.user_api_key_alias": "prod-key", + "langfuse.trace.metadata.user_api_key_user_id": "user-1", + } + def test_request_metadata_keys_are_absent_without_standard_logging_object( self, monkeypatch: pytest.MonkeyPatch ) -> None: From e5a2b2f5ca7b12f09736950b351a72ab3adb73dd Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 09:01:35 +0000 Subject: [PATCH 09/28] test(langfuse_otel): set redaction flag directly instead of monkeypatching Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/test_langfuse_otel.py | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 720b81712f4..98fd7ebf41d 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -484,10 +484,7 @@ class TestLangfuseOtelIntegration: assert isinstance(config, OpenTelemetryConfig) # Endpoint assertion removed as side effect is gone - def test_request_metadata_is_emitted_under_langfuse_observation_and_trace_metadata_keys( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(litellm, "redact_user_api_key_info", False) + def test_request_metadata_is_emitted_under_langfuse_observation_and_trace_metadata_keys(self) -> None: request_metadata: Final = { "user_api_key_hash": "hash123", "user_api_key_alias": "prod-key", @@ -518,8 +515,9 @@ class TestLangfuseOtelIntegration: "langfuse.trace.metadata.user_api_key_team_alias": "team-a", } - def test_request_metadata_redaction_matches_vanilla_langfuse(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "redact_user_api_key_info", True) + def test_request_metadata_redaction_matches_vanilla_langfuse(self) -> None: + previous_flag: Final = litellm.redact_user_api_key_info + litellm.redact_user_api_key_info = True request_metadata: Final = { "user_api_key_hash": "hash123", "user_api_key_alias": "prod-key", @@ -536,22 +534,22 @@ class TestLangfuseOtelIntegration: "standard_logging_object": {"metadata": request_metadata}, } - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + try: + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - assert json.loads(actual["langfuse.observation.metadata"]) == { - "spend_logs_metadata": {"env": "prod"}, - "requester_ip_address": "10.0.0.1", - "requester_metadata": {"ticket": "LIT-8283"}, - } - assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] + assert json.loads(actual["langfuse.observation.metadata"]) == { + "spend_logs_metadata": {"env": "prod"}, + "requester_ip_address": "10.0.0.1", + "requester_metadata": {"ticket": "LIT-8283"}, + } + assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] + finally: + litellm.redact_user_api_key_info = previous_flag - def test_request_metadata_drops_null_fields_and_skips_empty_trace_identities( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(litellm, "redact_user_api_key_info", False) + def test_request_metadata_drops_null_fields_and_skips_empty_trace_identities(self) -> None: request_metadata: Final = { "user_api_key_alias": "prod-key", "user_api_key_user_id": "user-1", @@ -584,10 +582,7 @@ class TestLangfuseOtelIntegration: "langfuse.trace.metadata.user_api_key_user_id": "user-1", } - def test_request_metadata_keys_are_absent_without_standard_logging_object( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(litellm, "redact_user_api_key_info", False) + def test_request_metadata_keys_are_absent_without_standard_logging_object(self) -> None: kwargs: Final = {"litellm_params": {"metadata": {"trace_metadata": {"k": "v"}}}} with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: From 049c629cc36aa76222ac726d9d9b804575bce368 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 10:27:40 +0000 Subject: [PATCH 10/28] test(langfuse_otel): add integration audit matrix for langfuse metadata keys Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/contracts.json | 51 + .../test_langfuse_otel_metadata.py | 996 ++++++++++++++++-- 2 files changed, 967 insertions(+), 80 deletions(-) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index b9c8245f51f..c90835ad118 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -263,6 +263,57 @@ "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse": [ "other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla" ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream": [ + "other.observability.langfuse_otel.request_metadata_openai_sdk_stream" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async": [ + "other.observability.langfuse_otel.request_metadata_openai_sdk_async" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages": [ + "other.observability.langfuse_otel.request_metadata_anthropic_sdk_messages" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream": [ + "other.observability.langfuse_otel.request_metadata_anthropic_sdk_stream" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_api": [ + "other.observability.langfuse_otel.request_metadata_responses_api" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream": [ + "other.observability.langfuse_otel.request_metadata_responses_stream_sse" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_caller_trace_metadata_survives_flattened_identity": [ + "other.observability.langfuse_otel.caller_trace_metadata_preserved_with_flattened_identity" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_failure_span_carries_request_metadata": [ + "other.observability.langfuse_otel.failure_span_carries_request_metadata" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_emits_no_team_identity": [ + "other.observability.langfuse_otel.teamless_key_omits_team_identity" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_keeps_caller_trace_metadata": [ + "other.observability.langfuse_otel.teamless_key_keeps_caller_trace_metadata" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_values": [ + "other.observability.langfuse_otel.spend_logs_metadata_roundtrips_nonstring_values" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_unauthenticated_request_emits_no_span": [ + "other.observability.langfuse_otel.unauthenticated_request_emits_no_span" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_repeated_identical_requests_emit_one_span_each": [ + "other.observability.langfuse_otel.repeated_requests_each_emit_one_span" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once": [ + "other.observability.langfuse_otel.sink_outage_burst_lands_each_id_once" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports": [ + "other.observability.langfuse_otel.worker_kill_burst_keeps_serving" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_slow_sink_does_not_deadlock_exports": [ + "other.observability.langfuse_otel.slow_sink_does_not_deadlock_exports" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_v2_mapper_keeps_existing_trace_metadata_keys": [ + "other.observability.langfuse_otel.v2_mapper_unchanged" + ], "tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": [ "other.observability.guardrails.rewrite_reaches_correct_anthropic_positions" ], diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index bc22f187528..1a7ffd0e8d3 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -1,17 +1,42 @@ +import base64 import json +import threading +import time import uuid -from collections.abc import Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor, wait from contextlib import contextmanager +from dataclasses import dataclass from pathlib import Path from typing import Final +import httpx +import psutil import pytest import yaml -from integration._support.client import Gateway, eventually -from integration._support.process import owned_proxy +from anthropic import Anthropic, AsyncAnthropic +from integration._support.client import Gateway, Scenario, eventually +from integration._support.process import OwnedProxy, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server +from openai import AsyncOpenAI, OpenAI from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +_IDENTITIES: Final = ( + "user_api_key_alias", + "user_api_key_user_id", + "user_api_key_end_user_id", + "user_api_key_team_id", + "user_api_key_team_alias", +) + + +@dataclass(frozen=True, slots=True) +class _Rig: + candidate: Gateway + provider: Wire + collector: Wire + proxy: OwnedProxy + def _span_attributes(body: bytes) -> tuple[dict[str, object], ...]: request: Final = ExportTraceServiceRequest.FromString(body) @@ -23,38 +48,195 @@ def _span_attributes(body: bytes) -> tuple[dict[str, object], ...]: ) +def _sse_chat(response_id: str) -> tuple[bytes, ...]: + chunk: Final = { + "id": response_id, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + } + return ( + f"data: {json.dumps({**chunk, 'choices': [{'index': 0, 'delta': {'role': 'assistant', 'content': 'langfuse-echo'}, 'finish_reason': None}]})}\n\n".encode(), + f"data: {json.dumps({**chunk, 'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'stop'}]})}\n\n".encode(), + f"data: {json.dumps({**chunk, 'choices': [], 'usage': {'prompt_tokens': 3, 'completion_tokens': 2, 'total_tokens': 5}})}\n\n".encode(), + b"data: [DONE]\n\n", + ) + + +def _sse_messages(response_id: str) -> tuple[bytes, ...]: + def event(name: str, payload: Mapping[str, object]) -> bytes: + return f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() + + message: Final = { + "id": response_id, + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 3, "output_tokens": 1}, + } + return ( + event("message_start", {"type": "message_start", "message": message}), + event( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + event( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "langfuse-echo"}}, + ), + event("content_block_stop", {"type": "content_block_stop", "index": 0}), + event( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 2}}, + ), + event("message_stop", {"type": "message_stop"}), + ) + + +def _sse_responses(response_id: str) -> tuple[bytes, ...]: + def event(name: str, payload: Mapping[str, object]) -> bytes: + return f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() + + response: Final = { + "id": response_id, + "object": "response", + "model": "gpt-4o-mini", + "status": "completed", + "output": [ + { + "type": "message", + "id": f"item-{response_id}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "langfuse-echo"}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + } + return ( + event( + "response.created", + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + ), + event( + "response.output_text.delta", + { + "type": "response.output_text.delta", + "response_id": response_id, + "item_id": f"item-{response_id}", + "output_index": 0, + "content_index": 0, + "delta": "langfuse-echo", + }, + ), + event("response.completed", {"type": "response.completed", "response": response}), + ) + + +def _chat_body(response_id: str) -> dict[str, object]: + return { + "id": response_id, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "langfuse-echo"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + + +def _message_body(response_id: str) -> dict[str, object]: + return { + "id": response_id, + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "langfuse-echo"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 2}, + } + + +_RESPONSE_FIELDS: Final = { + "created_at": 1, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": {}, + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "max_output_tokens": None, + "previous_response_id": None, + "store": True, + "truncation": "disabled", + "user": None, +} + + +def _response_body(response_id: str) -> dict[str, object]: + return { + **_RESPONSE_FIELDS, + "id": response_id, + "object": "response", + "model": "gpt-4o-mini", + "status": "completed", + "output": [ + { + "type": "message", + "id": f"item-{response_id}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "langfuse-echo"}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + } + + @contextmanager def _langfuse_rig( gateway: Gateway, tmp_path: Path, settings: Mapping[str, object], marker: str, -) -> Iterator[tuple[Gateway, Wire, Wire]]: + *, + sink_reply: Callable[[Request], Reply] | None = None, + overrides: Mapping[str, str] | None = None, +) -> Iterator[_Rig]: def upstream(request: Request) -> Reply: if request.method == "GET": return Reply(body=b'{"data":[]}') - assert request.target.endswith("/chat/completions"), request.target - return Reply( - body=json.dumps( - { - "id": f"chatcmpl-{marker}", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "langfuse-echo"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, - } - ).encode() - ) + body: Final = json.loads(request.body or b"{}") + suffix: Final = uuid.uuid4().hex[:8] + if "-fail-" in json.dumps(body): + return Reply( + status=500, body=b'{"error": {"message": "scripted provider failure", "type": "server_error"}}' + ) + if request.target.endswith("/chat/completions"): + chat_id: Final = f"chatcmpl-{marker}-{suffix}" + if body.get("stream"): + return Reply(content_type="text/event-stream", chunks=_sse_chat(chat_id)) + return Reply(body=json.dumps(_chat_body(chat_id)).encode()) + if request.target.endswith("/messages"): + message_id: Final = f"msg-{marker}-{suffix}" + if body.get("stream"): + return Reply(content_type="text/event-stream", chunks=_sse_messages(message_id)) + return Reply(body=json.dumps(_message_body(message_id)).encode()) + if request.target.endswith("/responses"): + resp_id: Final = f"resp-{marker}-{suffix}" + if body.get("stream"): + return Reply(content_type="text/event-stream", chunks=_sse_responses(resp_id)) + return Reply(body=json.dumps(_response_body(resp_id)).encode()) + raise AssertionError(request.target) - def sink(_request: Request) -> Reply: - return Reply() + def sink(request: Request) -> Reply: + return sink_reply(request) if sink_reply is not None else Reply() with wire_server(upstream) as provider, wire_server(sink) as collector: raw_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) @@ -64,18 +246,49 @@ def _langfuse_rig( } path: Final = tmp_path / "langfuse_otel.yaml" path.write_text(yaml.safe_dump(config)) - with owned_proxy( + with owned_proxy_process( gateway, tmp_path, { "LANGFUSE_PUBLIC_KEY": "pk-lf-integration", "LANGFUSE_SECRET_KEY": "sk-lf-integration", "LANGFUSE_HOST": collector.url, + **{ + name: (collector.url if value == "__collector__" else value) + for name, value in dict(overrides or {}).items() + }, }, config=path, workers=2, - ) as candidate: - yield candidate, provider, collector + ) as owned: + yield _Rig(owned.gateway, provider, collector, owned) + + +def _span_response_id(span: Mapping[str, object]) -> str | None: + value: Final = span.get("llm.response.id") + if not isinstance(value, str): + return None + if value.startswith("resp_"): + try: + decoded: Final = base64.b64decode(value[5:] + "===").decode() + except Exception: + return value + marker, _, suffix = decoded.partition("response_id:") + if suffix: + return suffix.split(";", 1)[0] + return value + + +def _watched_spans(collector: Wire) -> Callable[[], tuple[dict[str, object], ...]]: + drained: list[ + dict[str, object] + ] = [] # mutable-ok: drain() consumes the wire queue, so observed spans must accumulate across eventually polls + + def snapshot() -> tuple[dict[str, object], ...]: + drained.extend(attributes for batch in collector.drain() for attributes in _span_attributes(batch.body)) + return tuple(drained) + + return snapshot def _observation_span(collector: Wire, response_id: str) -> dict[str, object]: @@ -84,39 +297,99 @@ def _observation_span(collector: Wire, response_id: str) -> dict[str, object]: attributes for batch in collector.drain() for attributes in _span_attributes(batch.body) - if attributes.get("llm.response.id") == response_id + if _span_response_id(attributes) == response_id ) observed: Final = eventually(spans, lambda values: len(values) == 1, seconds=25) return observed[0] +def _marked_span(collector: Wire, marker: str, needle: str) -> dict[str, object]: + def spans() -> tuple[dict[str, object], ...]: + return tuple( + attributes + for batch in collector.drain() + for attributes in _span_attributes(batch.body) + if needle in json.dumps(attributes, default=str) and marker in json.dumps(attributes, default=str) + ) + + observed: Final = eventually(spans, lambda values: len(values) == 1, seconds=25) + return observed[0] + + +def _scenario_identity( + scenario: Scenario, + provider: Wire, + *, + spend_logs_metadata: Mapping[str, object] | None = None, + team: bool = True, +) -> tuple[str, dict[str, str | None]]: + team_alias: Final = ("lf-team-" + uuid.uuid4().hex) if team else None + team_id: Final = scenario.team(team_alias=team_alias) if team else None + key_alias: Final = "lf-key-" + uuid.uuid4().hex + key: Final = scenario.key( + **({"team_id": team_id} if team_id else {}), + key_alias=key_alias, + metadata={"spend_logs_metadata": spend_logs_metadata or {"ticket": "LIT-8283"}}, + ) + model: Final = scenario.model(api_base=provider.url + "/v1") + end_user: Final = "lf-end-user-" + uuid.uuid4().hex + return key, { + "key_alias": key_alias, + "team_id": team_id, + "team_alias": team_alias, + "end_user": end_user, + "model": model, + } + + +def _assert_identity( + attrs: Mapping[str, object], + expected: Mapping[str, object], + *, + spend_logs_metadata: Mapping[str, object] | None = None, +) -> None: + assert "metadata" in attrs, sorted(attrs) + assert "langfuse.observation.metadata" in attrs, sorted(attrs) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + assert {key: observation.get(key) for key in expected} == dict(expected), observation + assert not [key for key, value in observation.items() if value is None], observation + if spend_logs_metadata is not None: + assert observation["spend_logs_metadata"] == dict(spend_logs_metadata), observation + flattened: Final = { + attribute: attrs.get(attribute) + for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) + if attribute in attrs + } + assert flattened == {f"langfuse.trace.metadata.{field}": value for field, value in expected.items()}, attrs + + +def _chat_identity_fields(identity: Mapping[str, str | None]) -> dict[str, object]: + return { + "user_api_key_alias": identity["key_alias"], + **({"user_api_key_team_id": identity["team_id"]} if identity["team_id"] else {}), + **({"user_api_key_team_alias": identity["team_alias"]} if identity["team_alias"] else {}), + **({"user_api_key_end_user_id": identity["end_user"]} if identity["end_user"] else {}), + } + + @pytest.mark.covers("other.observability.langfuse_otel.request_metadata_on_langfuse_keys") def test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys( gateway: Gateway, tmp_path: Path ) -> None: marker: Final = "lf-meta-" + uuid.uuid4().hex with ( - _langfuse_rig(gateway, tmp_path, {}, marker) as (candidate, provider, collector), - candidate.scenario() as scenario, + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, ): - team_alias: Final = "lf-team-" + uuid.uuid4().hex - team_id: Final = scenario.team(team_alias=team_alias) - key_alias: Final = "lf-key-" + uuid.uuid4().hex - key: Final = scenario.key( - team_id=team_id, - key_alias=key_alias, - metadata={"spend_logs_metadata": {"ticket": "LIT-8283"}}, - ) - model: Final = scenario.model(api_base=provider.url + "/v1") - end_user: Final = "lf-end-user-" + uuid.uuid4().hex - response: Final = candidate.request( + key, identity = _scenario_identity(scenario, rig.provider) + response: Final = rig.candidate.request( "POST", "/v1/chat/completions", { - "model": model, + "model": identity["model"], "messages": [{"role": "user", "content": marker}], - "user": end_user, + "user": identity["end_user"], "cache": {"no-cache": True}, }, key=key, @@ -124,33 +397,8 @@ def test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_tra assert response.status_code == 200, response.text assert response.json()["choices"][0]["message"]["content"] == "langfuse-echo" - attrs: Final = _observation_span(collector, response.json()["id"]) - assert "metadata" in attrs, sorted(attrs) - assert json.loads(str(attrs["metadata"]))["user_api_key_alias"] == key_alias - assert "langfuse.observation.metadata" in attrs, sorted(attrs) - observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) - expected: Final = { - "user_api_key_alias": key_alias, - "user_api_key_team_id": team_id, - "user_api_key_team_alias": team_alias, - "user_api_key_end_user_id": end_user, - } - assert {key_: observation.get(key_) for key_ in expected} == expected, observation - assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation - assert { - attribute: attrs.get(attribute) - for attribute in ( - "langfuse.trace.metadata.user_api_key_alias", - "langfuse.trace.metadata.user_api_key_team_id", - "langfuse.trace.metadata.user_api_key_team_alias", - "langfuse.trace.metadata.user_api_key_end_user_id", - ) - } == { - "langfuse.trace.metadata.user_api_key_alias": key_alias, - "langfuse.trace.metadata.user_api_key_team_id": team_id, - "langfuse.trace.metadata.user_api_key_team_alias": team_alias, - "langfuse.trace.metadata.user_api_key_end_user_id": end_user, - }, attrs + attrs: Final = _observation_span(rig.collector, response.json()["id"]) + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) @pytest.mark.covers("other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla") @@ -159,19 +407,15 @@ def test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfus ) -> None: marker: Final = "lf-redact-" + uuid.uuid4().hex with ( - _langfuse_rig(gateway, tmp_path, {"redact_user_api_key_info": True}, marker) as ( - candidate, - provider, - collector, - ), - candidate.scenario() as scenario, + _langfuse_rig(gateway, tmp_path, {"redact_user_api_key_info": True}, marker) as rig, + rig.candidate.scenario() as scenario, ): key: Final = scenario.key( key_alias="lf-redact-" + uuid.uuid4().hex, metadata={"spend_logs_metadata": {"ticket": "LIT-8283"}}, ) - model: Final = scenario.model(api_base=provider.url + "/v1") - response: Final = candidate.request( + model: Final = scenario.model(api_base=rig.provider.url + "/v1") + response: Final = rig.candidate.request( "POST", "/v1/chat/completions", { @@ -183,9 +427,601 @@ def test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfus ) assert response.status_code == 200, response.text - attrs: Final = _observation_span(collector, response.json()["id"]) + attrs: Final = _observation_span(rig.collector, response.json()["id"]) assert "langfuse.observation.metadata" in attrs, sorted(attrs) observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) assert not [name for name in observation if name.startswith("user_api_key")], observation + assert not [key_ for key_, value in observation.items() if value is None], observation assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation assert not [name for name in attrs if name.startswith("langfuse.trace.metadata.")], sorted(attrs) + + +@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_openai_sdk_stream") +def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-sdkstream-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + client: Final = OpenAI( + base_url=str(rig.candidate.client.base_url).rstrip("/") + "/v1", + api_key=key, + http_client=httpx.Client(trust_env=False, timeout=30), + ) + chunks: Final = tuple( + client.chat.completions.create( + model=identity["model"], + messages=[{"role": "user", "content": marker}], + stream=True, + user=identity["end_user"], + extra_body={"cache": {"no-cache": True}}, + ) + ) + assert chunks, "stream produced no chunks" + response_id: Final = chunks[-1].id + attrs: Final = _observation_span(rig.collector, response_id) + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + + +@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_openai_sdk_async") +async def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-sdkasync-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + client: Final = AsyncOpenAI( + base_url=str(rig.candidate.client.base_url).rstrip("/") + "/v1", + api_key=key, + http_client=httpx.AsyncClient(trust_env=False, timeout=30), + ) + response: Final = await client.chat.completions.create( + model=identity["model"], + messages=[{"role": "user", "content": marker}], + user=identity["end_user"], + extra_body={"cache": {"no-cache": True}}, + ) + attrs: Final = _observation_span(rig.collector, response.id) + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + + +@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_anthropic_sdk_messages") +def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-anthropic-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) + client: Final = Anthropic( + base_url=str(rig.candidate.client.base_url).rstrip("/"), + api_key=key, + http_client=httpx.Client(trust_env=False, timeout=30), + ) + message: Final = client.messages.create( + model=model, + max_tokens=64, + messages=[{"role": "user", "content": marker}], + extra_body={"litellm_metadata": {"tags": ["lf8283"]}, "metadata": {"user_id": identity["end_user"]}}, + ) + attrs: Final = _observation_span(rig.collector, message.id) + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + + +@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_anthropic_sdk_stream") +async def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-anthstream-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) + client: Final = AsyncAnthropic( + base_url=str(rig.candidate.client.base_url).rstrip("/"), + api_key=key, + http_client=httpx.AsyncClient(trust_env=False, timeout=30), + ) + async with client.messages.stream( + model=model, + max_tokens=64, + messages=[{"role": "user", "content": marker}], + extra_body={"metadata": {"user_id": identity["end_user"]}}, + ) as stream: + final: Final = await stream.get_final_message() + attrs: Final = _observation_span(rig.collector, final.id) + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + + +@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_responses_api") +def test_langfuse_otel_metadata_reaches_langfuse_for_responses_api(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-responses-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + client: Final = OpenAI( + base_url=str(rig.candidate.client.base_url).rstrip("/") + "/v1", + api_key=key, + http_client=httpx.Client(trust_env=False, timeout=30), + ) + response: Final = client.responses.create( + model=identity["model"], + input=marker, + user=identity["end_user"], + extra_body={"cache": {"no-cache": True}}, + ) + attrs: Final = _observation_span(rig.collector, response.id) + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + + +@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_responses_stream_sse") +def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-respstream-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + with rig.candidate.client.stream( + "POST", + "/v1/responses", + json={ + "model": identity["model"], + "input": marker, + "user": identity["end_user"], + "stream": True, + "cache": {"no-cache": True}, + }, + headers={"Authorization": f"Bearer {key}"}, + ) as stream: + events: Final = tuple( + json.loads(line.removeprefix("data: ")) + for line in stream.iter_lines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + completed: Final = tuple(event for event in events if event.get("type") == "response.completed") + assert completed, events + attrs: Final = _marked_span(rig.collector, marker, "langfuse.observation.metadata") + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + + +@pytest.mark.covers("other.observability.langfuse_otel.caller_trace_metadata_preserved_with_flattened_identity") +def test_langfuse_otel_caller_trace_metadata_survives_flattened_identity(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-deploy-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "user": identity["end_user"], + "metadata": {"trace_metadata": {"deploy": "blue"}, "tags": ["lf8283"]}, + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + attrs: Final = _observation_span(rig.collector, response.json()["id"]) + assert json.loads(str(attrs["langfuse.trace.metadata"])) == {"deploy": "blue"}, attrs + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + + +@pytest.mark.covers("other.observability.langfuse_otel.failure_span_carries_request_metadata") +def test_langfuse_otel_failure_span_carries_request_metadata(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-fail-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": f"{marker} scripted -fail- upstream"}], + "user": identity["end_user"], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code >= 500, response.text + attrs: Final = _marked_span(rig.collector, marker, "langfuse.observation.") + _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + + +@pytest.mark.covers("other.observability.langfuse_otel.teamless_key_omits_team_identity") +def test_langfuse_otel_teamless_key_emits_no_team_identity(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-noteam-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider, team=False) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + attrs: Final = _observation_span(rig.collector, response.json()["id"]) + assert "langfuse.observation.metadata" in attrs, sorted(attrs) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + for field in ("user_api_key_team_id", "user_api_key_team_alias", "user_api_key_end_user_id"): + assert field not in observation, observation + assert not [key_ for key_, value in observation.items() if value is None], observation + assert { + attribute: attrs.get(attribute) + for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) + if attribute in attrs + } == {"langfuse.trace.metadata.user_api_key_alias": identity["key_alias"]}, attrs + + +@pytest.mark.covers("other.observability.langfuse_otel.teamless_key_keeps_caller_trace_metadata") +def test_langfuse_otel_teamless_key_keeps_caller_trace_metadata(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-noteam-meta-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider, team=False) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "metadata": {"trace_metadata": {"deploy": "blue"}}, + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + attrs: Final = _observation_span(rig.collector, response.json()["id"]) + assert json.loads(str(attrs["langfuse.trace.metadata"])) == {"deploy": "blue"}, attrs + assert "langfuse.trace.metadata.user_api_key_team_id" not in attrs, attrs + assert "langfuse.trace.metadata.user_api_key_team_alias" not in attrs, attrs + assert attrs.get("langfuse.trace.metadata.user_api_key_alias") == identity["key_alias"], attrs + + +@pytest.mark.covers("other.observability.langfuse_otel.spend_logs_metadata_roundtrips_nonstring_values") +def test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_values( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-bigmeta-" + uuid.uuid4().hex + spend_logs_metadata: Final = { + "blob": "x" * 5000, + "count": 7, + "steps": ["a", "b"], + "nested": {"inner": {"leaf": True}}, + } + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider, spend_logs_metadata=spend_logs_metadata) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "user": identity["end_user"], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + attrs: Final = _observation_span(rig.collector, response.json()["id"]) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + assert observation["spend_logs_metadata"] == dict(spend_logs_metadata), observation + _assert_identity(attrs, _chat_identity_fields(identity)) + + +@pytest.mark.covers("other.observability.langfuse_otel.unauthenticated_request_emits_no_span") +def test_langfuse_otel_unauthenticated_request_emits_no_span(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-unauth-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + denied: Final = rig.candidate.client.post( + "/v1/chat/completions", + json={ + "model": identity["model"], + "messages": [{"role": "user", "content": f"{marker}-denied"}], + }, + headers={}, + ) + assert denied.status_code == 401, denied.text + watch: Final = _watched_spans(rig.collector) + first: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert first.status_code == 200, first.text + first_id: Final = first.json()["id"] + eventually(watch, lambda spans: sum(_span_response_id(s) == first_id for s in spans) == 1, seconds=25) + second: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker + "-settle"}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert second.status_code == 200, second.text + second_id: Final = second.json()["id"] + settled: Final = eventually( + watch, lambda spans: sum(_span_response_id(s) == second_id for s in spans) == 1, seconds=25 + ) + for span in (span for span in settled if f"{marker}-denied" in json.dumps(span, default=str)): + assert not [name for name in span if name.startswith("langfuse.trace.metadata.user_api_key")], span + leaked: Final = json.loads(str(span.get("langfuse.observation.metadata", "{}"))) + for field in ( + "user_api_key_alias", + "user_api_key_team_id", + "user_api_key_team_alias", + "user_api_key_end_user_id", + ): + assert field not in leaked, leaked + + +@pytest.mark.covers("other.observability.langfuse_otel.repeated_requests_each_emit_one_span") +def test_langfuse_otel_repeated_identical_requests_emit_one_span_each(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-repeat-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + watch: Final = _watched_spans(rig.collector) + responses: Final = tuple( + rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "cache": {"no-cache": True}, + }, + key=key, + ) + for _ in range(3) + ) + assert all(response.status_code == 200 for response in responses), [r.text for r in responses] + response_ids: Final = tuple(response.json()["id"] for response in responses) + assert len(set(response_ids)) == 3, response_ids + settled: Final = eventually( + watch, + lambda spans: all(sum(_span_response_id(s) == rid for s in spans) == 1 for rid in response_ids), + seconds=40, + ) + for rid in response_ids: + assert sum(_span_response_id(s) == rid for s in settled) == 1 + + +@pytest.mark.covers("other.observability.langfuse_otel.sink_outage_burst_lands_each_id_once") +def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-chaos-outage-" + uuid.uuid4().hex + outage: Final = threading.Event() + + def sink_reply(request: Request) -> Reply: + if outage.is_set(): + return Reply(status=503, body=b'{"error": "sink outage"}') + return Reply() + + with ( + _langfuse_rig(gateway, tmp_path, {}, marker, sink_reply=sink_reply) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + anthropic_model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) + watch: Final = _watched_spans(rig.collector) + + def call(index: int) -> str: + path: Final = ("/v1/chat/completions", "/v1/messages", "/v1/responses")[index % 3] + streaming: Final = index % 2 == 0 + body: Final = { + "/v1/chat/completions": { + "model": identity["model"], + "messages": [{"role": "user", "content": f"{marker}-{index}"}], + "stream": streaming, + "cache": {"no-cache": True}, + }, + "/v1/messages": { + "model": anthropic_model, + "messages": [{"role": "user", "content": f"{marker}-{index}"}], + "max_tokens": 64, + "stream": streaming, + }, + "/v1/responses": {"model": identity["model"], "input": f"{marker}-{index}", "stream": streaming}, + }[path] + if streaming: + with rig.candidate.client.stream( + "POST", path, json=body, headers={"Authorization": f"Bearer {key}"} + ) as stream: + assert stream.status_code == 200, f"burst {index}: {stream.status_code}" + events: Final = tuple( + line.removeprefix("data: ") + for line in stream.iter_lines() + if line.startswith("data: ") and not line.endswith("[DONE]") + ) + payloads: Final = tuple(json.loads(line) for line in events) + found: Final = next( + payload.get("response_id") or (payload.get("message") or {}).get("id") or payload.get("id") + for payload in payloads + if payload.get("response_id") or payload.get("message") or payload.get("id") + ) + return str(found) + response: Final = rig.candidate.client.post(path, json=body, headers={"Authorization": f"Bearer {key}"}) + assert response.status_code == 200, f"burst {index}: {response.text}" + return response.json()["id"] + + with ThreadPoolExecutor(max_workers=30) as pool: + futures: Final = tuple(pool.submit(call, index) for index in range(30)) + while sum(f.done() for f in futures) < 10: + wait(futures, timeout=0.05) + outage.set() + while sum(f.done() for f in futures) < 25: + wait(futures, timeout=0.05) + outage.clear() + response_ids: Final = tuple(f.result() for f in futures) + assert len(set(response_ids)) == 30, response_ids + settled: Final = eventually( + watch, + lambda spans: all(sum(_span_response_id(s) == rid for s in spans) == 1 for rid in response_ids), + seconds=80, + ) + assert sorted(_span_response_id(span) for span in settled if _span_response_id(span) in response_ids) == sorted( + response_ids + ) + + +@pytest.mark.covers("other.observability.langfuse_otel.worker_kill_burst_keeps_serving") +def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-chaos-kill-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + watch: Final = _watched_spans(rig.collector) + + def call(index: int) -> tuple[int, str]: + try: + response: Final = rig.candidate.client.post( + "/v1/chat/completions", + json={ + "model": identity["model"], + "messages": [{"role": "user", "content": f"{marker}-{index}"}], + "cache": {"no-cache": True}, + }, + headers={"Authorization": f"Bearer {key}"}, + timeout=30, + ) + except httpx.HTTPError as error: + return 0, repr(error) + return response.status_code, response.json()["id"] if response.status_code == 200 else response.text + + workers: Final = tuple(psutil.Process(rig.proxy.process.pid).children(recursive=True)) + assert workers, "owned proxy reported no worker children" + with ThreadPoolExecutor(max_workers=20) as pool: + futures: Final = tuple(pool.submit(call, index) for index in range(20)) + while sum(f.done() for f in futures) < 5: + wait(futures, timeout=0.05) + workers[0].kill() + outcomes: Final = tuple(f.result() for f in futures) + failures: Final = tuple(status for status, _ in outcomes if status != 200) + succeeded: Final = tuple(identifier for status, identifier in outcomes if status == 200) + assert len(set(succeeded)) == len(succeeded), outcomes + settled: Final = eventually( + watch, + lambda spans: all(sum(_span_response_id(s) == rid for s in spans) == 1 for rid in succeeded), + seconds=60, + ) + assert not failures, f"worker kill burst lost caller responses: {failures}" + assert len(settled) >= 20 and len(succeeded) == 20 + + +@pytest.mark.covers("other.observability.langfuse_otel.slow_sink_does_not_deadlock_exports") +def test_langfuse_otel_slow_sink_does_not_deadlock_exports(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-chaos-slow-" + uuid.uuid4().hex + + def sink_reply(request: Request) -> Reply: + time.sleep(2) + return Reply() + + with ( + _langfuse_rig(gateway, tmp_path, {}, marker, sink_reply=sink_reply) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + watch: Final = _watched_spans(rig.collector) + responses: Final = tuple( + rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": f"{marker}-{index}"}], + "cache": {"no-cache": True}, + }, + key=key, + ) + for index in range(10) + ) + assert all(response.status_code == 200 for response in responses), [r.text for r in responses] + response_ids: Final = tuple(response.json()["id"] for response in responses) + settled: Final = eventually( + watch, + lambda spans: all(sum(_span_response_id(s) == rid for s in spans) == 1 for rid in response_ids), + seconds=60, + ) + assert sorted(_span_response_id(span) for span in settled if _span_response_id(span) in response_ids) == sorted( + response_ids + ) + + +@pytest.mark.covers("other.observability.langfuse_otel.v2_mapper_unchanged") +def test_langfuse_otel_v2_mapper_keeps_existing_trace_metadata_keys(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-v2-" + uuid.uuid4().hex + with ( + _langfuse_rig( + gateway, + tmp_path, + {}, + marker, + overrides={"LITELLM_OTEL_V2": "1"}, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "user": identity["end_user"], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + watch: Final = _watched_spans(rig.collector) + observed: Final = eventually( + watch, + lambda spans: sum(s.get("gen_ai.response.id") == response.json()["id"] for s in spans) == 1, + seconds=25, + ) + attrs: Final = next(s for s in observed if s.get("gen_ai.response.id") == response.json()["id"]) + assert attrs.get("langfuse.trace.metadata.team_id") == identity["team_id"], attrs + assert attrs.get("langfuse.trace.metadata.team_alias") == identity["team_alias"], attrs + assert "langfuse.observation.metadata" not in attrs, sorted(attrs) From 55de24df00e2786e373f79e850f79d351e31e6a0 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 10:32:40 +0000 Subject: [PATCH 11/28] test(langfuse_otel): tighten audit chaos and unauthenticated cells Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/contracts.json | 2095 +---------------- .../test_langfuse_otel_metadata.py | 57 +- 2 files changed, 37 insertions(+), 2115 deletions(-) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index c90835ad118..c9ec0a6c1e9 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1,2094 +1 @@ -{ - "groups": { - "management": [ - "management", - "authorization", - "configuration" - ], - "accounting": [ - "pricing", - "spend" - ], - "database": [ - "database" - ], - "providers": [ - "providers", - "routing", - "streaming" - ], - "extensions": [ - "mcp", - "observability", - "compatibility" - ], - "sdk": [ - "sdk" - ], - "cost": [ - "cost_calculation" - ] - }, - "tests": { - "tests/integration/management/test_key_updates.py::test_update_preserves_independent_fields_and_serving": [ - "mgmt.key.update.preserves_independent_fields" - ], - "tests/integration/pricing/test_configured_prices.py::test_custom_price_is_reported_and_charged": [ - "quota_management.spend_tracking.custom_price.matches_input_rates" - ], - "tests/integration/providers/test_request_boundary.py::test_internal_request_state_does_not_reach_provider": [ - "other.provider_wire.internal_parameters_filtered" - ], - "tests/integration/pricing/test_configured_prices.py::test_default_prices_survive_nullable_sibling_and_reload": [ - "quota_management.spend_tracking.default_prices.survive_nullable_sibling_reload" - ], - "tests/integration/providers/test_request_boundary.py::test_upstream_rejects_corruption_and_accepts_supported_metadata": [ - "other.provider_wire.validator_rejects_corruption" - ], - "tests/integration/pricing/test_configured_prices.py::test_loaded_router_preserves_cached_defaults_during_real_requests": [ - "quota_management.spend_tracking.default_prices.loaded_router_preserves_cached_defaults" - ], - "tests/integration/management/test_partial_update_sequences.py::test_generated_partial_updates_preserve_persisted_and_effective_state": [ - "mgmt.key.update.generated_sequences_preserve_state" - ], - "tests/integration/management/test_partial_update_sequences.py::test_zero_false_and_empty_values_are_not_treated_as_omission": [ - "mgmt.key.update.false_zero_and_empty_values_affect_serving" - ], - "tests/integration/management/test_partial_update_sequences.py::test_project_omission_clear_and_invalid_update_have_distinct_effects": [ - "mgmt.key.update.project_clear_preserves_scope", - "mgmt.key.update.invalid_batch_is_atomic" - ], - "tests/integration/authorization/test_warmed_policy.py::test_generated_policy_changes_reach_both_warmed_workers": [ - "mgmt.key.update.two_workers_enforce_warmed_policy" - ], - "tests/integration/authorization/test_warmed_policy.py::test_scim_deactivation_blocks_null_and_false_keys_but_preserves_other_owners": [ - "mgmt.user.scim.deactivation_includes_nullable_blocked_keys" - ], - "tests/integration/authorization/test_warmed_policy.py::test_warmed_team_role_demotion_prevents_later_management_writes": [ - "mgmt.team.member_update.demoted_role_cannot_write" - ], - "tests/integration/configuration/test_effective_settings.py::test_model_block_changes_actual_route_and_leaves_other_route_working": [ - "mgmt.model.block.changes_serving_and_preserves_control" - ], - "tests/integration/configuration/test_effective_settings.py::test_saved_retry_setting_controls_real_attempts_and_restores": [ - "mgmt.router_settings.update.changes_observed_attempt_count" - ], - "tests/integration/configuration/test_effective_settings.py::test_credential_value_update_and_model_reload_reach_provider": [ - "mgmt.credential.update.saved_value_reaches_wire" - ], - "tests/integration/management/test_partial_update_sequences.py::test_denied_key_update_preserves_saved_grants_and_serving": [ - "mgmt.key.update.denied_request_preserves_effective_state" - ], - "tests/integration/authorization/test_warmed_policy.py::test_expiry_and_explicit_clear_reach_both_warmed_workers": [ - "mgmt.key.update.expiry_changes_reach_warmed_workers" - ], - "tests/integration/database/test_partition_transactions.py::test_real_partition_ddl_survives_witnessed_lock_and_is_idempotent": [ - "other.database.partitions.lock_wait_outlives_transaction_default", - "other.database.partitions.repeat_preserves_rows" - ], - "tests/integration/database/test_reader_writer_regeneration.py::test_key_regeneration_uses_writer_with_a_real_readonly_reader": [ - "other.database.regeneration.writer_updates_dependent_grants" - ], - "tests/integration/pricing/test_price_precedence.py::test_generated_zero_null_and_omitted_prices_follow_independent_arithmetic": [ - "quota_management.spend_tracking.price_precedence.zero_and_default_rates" - ], - "tests/integration/pricing/test_price_precedence.py::test_same_upstream_aliases_keep_distinct_prices_after_reload": [ - "quota_management.spend_tracking.alias_prices.remain_independent_on_reload" - ], - "tests/integration/pricing/test_off_peak_pricing.py::test_open_off_peak_window_bills_off_peak_rates": [ - "quota_management.spend_tracking.off_peak_pricing.open_window_bills_off_peak_rates" - ], - "tests/integration/pricing/test_off_peak_pricing.py::test_closed_off_peak_window_bills_standard_rates": [ - "quota_management.spend_tracking.off_peak_pricing.closed_window_bills_standard_rates" - ], - "tests/integration/spend/test_cache_and_quota.py::test_generated_cache_sequences_preserve_content_usage_and_zero_hit_cost": [ - "quota_management.response_cache.generated_sequences_preserve_content_and_accounting" - ], - "tests/integration/spend/test_cache_and_quota.py::test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores": [ - "quota_management.budget.key.boundary_blocks_before_provider_and_reset_restores" - ], - "tests/integration/spend/test_cache_and_quota.py::test_different_system_messages_do_not_share_a_cached_response": [ - "quota_management.response_cache.system_messages_partition_cache_identity" - ], - "tests/integration/database/test_transaction_atomicity.py::test_access_group_second_key_constraint_failure_rolls_back_all_writes": [ - "other.database.access_group.failed_second_write_rolls_back_first" - ], - "tests/integration/spend/test_cache_and_quota.py::test_repeated_hits_keep_response_identity_and_create_distinct_zero_cost_rows": [ - "quota_management.response_cache.repeated_hits_preserve_identity_and_single_charge" - ], - "tests/integration/providers/test_s3_wire.py::test_sigv4_verifier_matches_published_put_and_rejects_corruption": [ - "other.provider_wire.s3.verifier_known_answer_and_negative_controls" - ], - "tests/integration/providers/test_s3_wire.py::test_s3_sync_and_async_uploads_pass_independent_wire_verification": [ - "other.provider_wire.s3.sync_async_reserved_keys_are_signed_and_accepted" - ], - "tests/integration/providers/test_bedrock_auth_wire.py::test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials": [ - "other.provider_wire.bedrock.bearer_sdk_skips_credential_chain" - ], - "tests/integration/providers/test_bedrock_auth_wire.py::test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload": [ - "other.provider_wire.bedrock.bearer_db_yaml_survives_reload" - ], - "tests/integration/streaming/test_stream_contracts.py::test_generated_tcp_partitions_preserve_unicode_text_identity_and_final_usage": [ - "other.streaming.byte_partitions.preserve_text_identity_and_usage" - ], - "tests/integration/streaming/test_stream_contracts.py::test_fragmented_tool_names_and_arguments_keep_each_call_identity": [ - "other.streaming.tools.fragmented_calls_keep_independent_arguments" - ], - "tests/integration/streaming/test_stream_contracts.py::test_proxy_stream_usage_visibility_keeps_exact_persisted_charge": [ - "other.streaming.usage.client_visibility_preserves_persisted_accounting" - ], - "tests/integration/streaming/test_stream_contracts.py::test_truncated_http_stream_is_an_error_and_next_stream_succeeds": [ - "other.streaming.failure.truncated_transport_raises_and_control_recovers" - ], - "tests/integration/streaming/test_stream_contracts.py::test_client_cancellation_releases_the_actual_provider_connection": [ - "other.streaming.cancellation.closes_actual_provider_connection" - ], - "tests/integration/routing/test_observed_routing.py::test_retry_counts_and_public_errors_match_actual_provider_attempts": [ - "other.routing.retries.several_attempts_reach_success_without_hidden_retries", - "other.routing.errors.nonretryable_and_exhausted_failures_remain_errors" - ], - "tests/integration/routing/test_observed_routing.py::test_loaded_fallback_selects_expected_deployment_and_keeps_response_identity": [ - "other.routing.fallback.loaded_configuration_selects_only_permitted_target" - ], - "tests/integration/routing/test_observed_routing.py::test_saved_deployment_target_update_changes_wire_and_preserves_control": [ - "other.routing.alias_update.persisted_target_changes_only_selected_route" - ], - "tests/integration/providers/test_bedrock_role_configuration.py::test_role_reference_from_db_and_yaml_reaches_real_sts_http_and_bedrock": [ - "other.provider_wire.bedrock.db_yaml_role_reference_reaches_sts_and_signed_request" - ], - "tests/integration/routing/test_redis_recovery.py::test_owned_redis_outage_recovers_requests_and_real_response_cache": [ - "other.routing.redis.owned_outage_recovers_serving_and_response_cache" - ], - "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[type_word]": [ - "other.provider_wire.anthropic.bare_string_content_item_is_client_error" - ], - "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[plain]": [ - "other.provider_wire.anthropic.bare_string_content_item_is_client_error" - ], - "tests/integration/providers/test_anthropic_wire.py::test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contracts": [ - "other.provider_wire.anthropic.tool_history_system_cache_and_internal_fields", - "quota_management.spend_tracking.cache_tokens.disjoint_classes_use_explicit_rates" - ], - "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_create_status_and_content_follow_queue_wire_contract": [ - "other.provider_wire.fal_ai.video_queue_create_status_and_content_download" - ], - "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_failed_result_reports_failed_status_and_fal_error": [ - "other.provider_wire.fal_ai.video_failed_result_surfaces_fal_error" - ], - "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_auto_duration_omits_duration_and_queues": [ - "other.provider_wire.fal_ai.h3_auto_duration_omits_duration_and_queues" - ], - "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_oversized_size_uses_top_resolution_tier_and_queues": [ - "other.provider_wire.fal_ai.h3_oversized_size_uses_top_resolution_tier" - ], - "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": [ - "other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing" - ], - "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_prices_non_canonical_size_from_nearest_row": [ - "other.provider_wire.fal_ai.gpt_image_generation_noncanonical_size_uses_nearest_keyed_row" - ], - "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_sdk_response_honors_dump_options": [ - "other.provider_wire.fal_ai.sdk_image_response_dump_options" - ], - "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": [ - "other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing" - ], - "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_charges_and_polls_pass_through_free": [ - "other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not" - ], - "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_prices_string_resolution_like_the_integer": [ - "other.provider_wire.fal_ai.passthrough_queue_submit_prices_string_resolution_like_integer" - ], - "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_to_catalog_key_the_pricer_cannot_price_is_rejected_not_forwarded": [ - "other.provider_wire.fal_ai.passthrough_queue_submit_rejects_unpriceable_catalog_key" - ], - "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": [ - "other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing" - ], - "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_lora_depth_edit_sends_single_image_url_and_charges_flat_row": [ - "other.provider_wire.fal_ai.flux_lora_depth_edit_single_image_url_and_flat_pricing" - ], - "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_sends_prompt_image_and_reasoning": [ - "other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing" - ], - "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_rejects_non_string_reasoning_effort_before_the_wire": [ - "other.provider_wire.fal_ai.chat_non_string_reasoning_effort_rejected_before_wire" - ], - "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_without_deployment_api_base_uses_global_api_base": [ - "other.provider_wire.fal_ai.global_api_base_routes_image_generation" - ], - "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-pro]": [ - "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" - ], - "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-flash]": [ - "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" - ], - "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas": [ - "other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas" - ], - "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_tool_call_is_forwarded_and_returned": [ - "other.provider_wire.xiaomi_mimo.tool_call_survives_translation" - ], - "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_video_create_uses_canonical_body_and_status_path": [ - "other.provider_wire.fal_ai.video_queue_create_status_and_content_download" - ], - "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_carries_the_deployment_extra_headers": [ - "other.provider_wire.fal_ai.video_result_probe_forwards_extra_headers" - ], - "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_reuses_the_ssl_verify_false_client": [ - "other.provider_wire.fal_ai.video_result_probe_honors_ssl_verify" - ], - "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_provider_hanging_up_on_the_result_probe_keeps_the_completed_status": [ - "other.provider_wire.fal_ai.video_result_probe_hangup_stays_completed" - ], - "tests/integration/mcp/test_mcp_lifecycle.py::test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit": [ - "mcp.call_tool.saved_headers.reach_actual_transport" - ], - "tests/integration/mcp/test_mcp_lifecycle.py::test_tool_error_remains_error_and_healthy_sibling_returns_value": [ - "mcp.call_tool.errors.tool_failure_is_not_success" - ], - "tests/integration/mcp/test_mcp_lifecycle.py::test_generated_mcp_edits_preserve_actual_headers_and_tool_results": [ - "other.mcp.lifecycle.generated_save_reload_preserves_effective_headers" - ], - "tests/integration/observability/test_callback_delivery.py::test_concurrent_success_and_failure_join_callbacks_and_rows_without_credentials": [ - "other.observability.callbacks.credentials_stay_out_of_event_bodies", - "other.observability.callbacks.concurrent_results_join_complete_events_and_rows" - ], - "tests/integration/observability/test_otel_text_completion_choices.py::test_otel_weave_output_keeps_text_completion_provider_fields_beside_the_synthesized_message": [ - "other.observability.otel.text_completion_choices_keep_provider_fields" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys": [ - "other.observability.langfuse_otel.request_metadata_on_langfuse_keys" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse": [ - "other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream": [ - "other.observability.langfuse_otel.request_metadata_openai_sdk_stream" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async": [ - "other.observability.langfuse_otel.request_metadata_openai_sdk_async" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages": [ - "other.observability.langfuse_otel.request_metadata_anthropic_sdk_messages" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream": [ - "other.observability.langfuse_otel.request_metadata_anthropic_sdk_stream" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_api": [ - "other.observability.langfuse_otel.request_metadata_responses_api" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream": [ - "other.observability.langfuse_otel.request_metadata_responses_stream_sse" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_caller_trace_metadata_survives_flattened_identity": [ - "other.observability.langfuse_otel.caller_trace_metadata_preserved_with_flattened_identity" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_failure_span_carries_request_metadata": [ - "other.observability.langfuse_otel.failure_span_carries_request_metadata" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_emits_no_team_identity": [ - "other.observability.langfuse_otel.teamless_key_omits_team_identity" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_keeps_caller_trace_metadata": [ - "other.observability.langfuse_otel.teamless_key_keeps_caller_trace_metadata" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_values": [ - "other.observability.langfuse_otel.spend_logs_metadata_roundtrips_nonstring_values" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_unauthenticated_request_emits_no_span": [ - "other.observability.langfuse_otel.unauthenticated_request_emits_no_span" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_repeated_identical_requests_emit_one_span_each": [ - "other.observability.langfuse_otel.repeated_requests_each_emit_one_span" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once": [ - "other.observability.langfuse_otel.sink_outage_burst_lands_each_id_once" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports": [ - "other.observability.langfuse_otel.worker_kill_burst_keeps_serving" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_slow_sink_does_not_deadlock_exports": [ - "other.observability.langfuse_otel.slow_sink_does_not_deadlock_exports" - ], - "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_v2_mapper_keeps_existing_trace_metadata_keys": [ - "other.observability.langfuse_otel.v2_mapper_unchanged" - ], - "tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": [ - "other.observability.guardrails.rewrite_reaches_correct_anthropic_positions" - ], - "tests/integration/compatibility/test_a2a_wire_versions.py::test_a2a_versions_and_legacy_casing_preserve_real_wire_and_response": [ - "other.compatibility.a2a.supported_versions_preserve_literal_envelopes" - ], - "tests/integration/compatibility/test_persisted_toolsets.py::test_existing_toolset_format_loads_before_start_and_keeps_sibling_denied": [ - "other.compatibility.mcp.persisted_tool_names_survive_candidate_startup" - ], - "tests/integration/mcp/test_oauth_configuration.py::test_partial_discovery_and_unrelated_edit_keep_actual_authorization_destination": [ - "other.mcp.oauth.discovery_cannot_erase_configured_authorization_endpoint" - ], - "tests/integration/observability/test_guardrail_effects.py::test_guardrail_denial_prevents_provider_and_preserves_allowed_control": [ - "other.observability.guardrails.denial_prevents_provider_with_allowed_control" - ], - "tests/integration/mcp/test_mcp_protocol_errors.py::test_jsonrpc_error_and_malformed_tool_result_remain_errors": [ - "other.mcp.errors.protocol_and_malformed_results_cannot_be_empty_success" - ], - "tests/integration/compatibility/test_openai_consumer.py::test_retained_openai_clients_parse_real_proxy_tool_and_usage_responses": [ - "other.compatibility.openai.retained_client_parses_tools_and_usage" - ], - "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [ - "quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals" - ], - "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_written_by_the_final_flush": [ - "quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch" - ], - "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_row_lock_is_written_exactly_once": [ - "quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch" - ], - "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [ - "quota_management.spend_tracking.spend_calculate.rejects_unpriced_model" - ], - "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini-live-2.5-flash-preview-native-audio-09-2025]": [ - "quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate" - ], - "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini/gemini-live-2.5-flash-preview-native-audio-09-2025]": [ - "quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate" - ], - "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [ - "mgmt.key.update.project_detach_denied_to_restricted_actor" - ], - "tests/integration/management/test_partial_update_sequences.py::test_cross_tenant_actor_cannot_read_update_or_detach_project_key": [ - "mgmt.key.info.cross_tenant_key_is_denied", - "mgmt.key.update.cross_tenant_key_is_denied", - "mgmt.key.update.cross_tenant_project_detach_is_denied" - ], - "tests/integration/management/test_project_lifecycle.py::test_project_new_persists_real_state": [ - "mgmt.project.new.real_route_persists" - ], - "tests/integration/management/test_project_lifecycle.py::test_project_update_persists_real_state": [ - "mgmt.project.update.real_route_persists" - ], - "tests/integration/management/test_project_lifecycle.py::test_project_delete_with_attached_key_refuses_and_preserves_state": [ - "mgmt.project.delete.attached_key_refusal_preserves_state" - ], - "tests/integration/sdk/test_http2_wire.py::test_async_handler_negotiates_http2_only_when_enabled": [ - "other.sdk_wire.http2.async_handler_negotiates_h2_only_when_enabled" - ], - "tests/integration/sdk/test_http2_wire.py::test_sync_handler_negotiates_http2_only_when_enabled": [ - "other.sdk_wire.http2.sync_handler_negotiates_h2_only_when_enabled" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-halved_rates_when_map_has_no_batch_keys]": [ - "quota_management.spend_tracking.batch_costs.fallback_rates" - ], - "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-cached_input_halved]": [ - "quota_management.spend_tracking.batch_costs.cached_input" - ], - "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.4-batch-explicit_batch_rates_bill_cached_at_batch_input_rate]": [ - "quota_management.spend_tracking.batch_costs.explicit_rates" - ], - "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-all_requests_failed_zero_spend]": [ - "quota_management.spend_tracking.batch_costs.failed_requests" - ], - "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-single_turn_text_audio_cached]": [ - "quota_management.spend_tracking.realtime_costs.single_turn" - ], - "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-two_turns_summed_into_one_row]": [ - "quota_management.spend_tracking.realtime_costs.multiple_turns" - ], - "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-priced_from_session_created_model]": [ - "quota_management.spend_tracking.realtime_costs.session_model" - ], - "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-session_without_turns_zero_spend]": [ - "quota_management.spend_tracking.realtime_costs.session_without_turns" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_5m]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_1h]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_output]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_low]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_high]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_output]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_low]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_high]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_native_json]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_zero_spend]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_429_zero_spend]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_5m]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_1h]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-anthropic_us_inference]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_5m]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_1h]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_input_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_read_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_write_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_fast_mode]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_us_inference]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_5m]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_1h]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_input_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_read_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_write_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-anthropic_us_inference]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-fallback_cache_read_at_half_input_rate]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-audio_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-image_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_input_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_cache_read_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-google_maps_grounding]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-fallback_video_tokens_at_input_rate]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-prompt_blocked]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_prompt_blocked]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_output]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-video_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-web_search_per_prompt]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-google_maps_grounding]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_reasoning_at_output_rate]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_image_tokens_at_input_rate]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-prompt_blocked]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_prompt_blocked]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-audio_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-image_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-video_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_input_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_cache_read_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-google_maps_grounding]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-prompt_blocked]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_prompt_blocked]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_output]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-image_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-video_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-web_search_per_prompt]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-google_maps_grounding]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-prompt_blocked]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_prompt_blocked]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_low]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_high]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-file_search]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_incomplete]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_incomplete]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_unvalidated]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_unvalidated]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_output]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_low]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_high]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_low]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_high]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-file_search]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_incomplete]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_incomplete]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_unvalidated]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_unvalidated]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_output]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_low]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_high]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_read_at_input_rate]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_write_at_input_rate]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_5m]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_1h]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_input_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_read_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_write_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_image_input]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_response_model_override]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_tool_call]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-next-transcriptions-per-second]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-verbose-next-transcriptions-duration]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-4o-transcribe-next-transcriptions-tokens]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[nova-next-transcriptions-per-second]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-whisper-next-transcriptions-deployment]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-speech-per-character]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-hd-speech-per-character]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-tts-next-speech-deployment]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-standard]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-hd]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-wide]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-two]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-low]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[imagen-next-images-one]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[amazon-nova-canvas-next-images-one]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-edit]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embeddings-4-large-deployment]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-embeddings-v4]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-rerank-v4]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-embeddings-titan-v2]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-embeddings-v5]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-one]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-three]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-total-tokens-fallback]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks-embeddings-v1]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-embeddings-002]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-list]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-single]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-basic]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-n-best]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-stream-usage]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-3-large-dimensions]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-batch]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-single]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-token-array]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-completions-v1]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-embeddings-v1]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[vertex-embeddings-text-006]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_reasoning]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream_cache_read]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_incomplete]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_previous_response_id]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_web_search_medium]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-responses_file_search]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_flex]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_priority]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_5m]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_1h]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_web_search]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream_cache_read]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_tiered_input_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-messages_input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-messages_input_text]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-messages_cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-passthrough-generate_content_priced_via_gemini_key]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-passthrough-stream_generate_content_priced_via_vertex_key]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-passthrough-messages]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-passthrough-messages_cache_read]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse_stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_input]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_boundary_stays_lower_tier]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_second_tier]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_above_top_range]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_below_128k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_above_128k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_creation_1h_above_200k]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-provider_reported_cost]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-token_priced]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-no_search]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-prompt_cache_hit]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-no_cache_fields_bills_zero_cache]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-reasoning_folded_into_completion]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-live_search]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-provider_reported_cost]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-invoke-haiku-json]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-invoke-haiku-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-profile-base-model]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-eu-regional-key]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-apac-bare-fallback]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-nova-2-pro]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-mistral-large-3-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-ai-gpt-5.4-mini-latest]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-ai-gpt-5.4-mini-latest-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-pinned-gpt-5.4-mini-stream]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[groq-qwen-3.8-json]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[groq-qwen-3.8-stream_x_groq_recount]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-command-a-v2-tokens]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[mistral-medium-2604-json]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openai-deployment-pricing-override]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_400_zero_spend]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_401_zero_spend]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_stream_request_zero_spend]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_upstream_500_zero_spend]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_upstream_500_zero_spend]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-fallback_billed_to_answering_deployment]": [ - "quota_management.spend_tracking.routing.fallback_billing" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-n_2_choices]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-finish_reason_length]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_usage_in_empty_choices_chunk]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_usage_in_last_delta_chunk]": [ - "quota_management.spend_tracking.scripted_wire.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-unknown_model_response_model_unknown]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-unknown_model_response_model_known]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-chat_request_to_embedding_entry]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-client_disconnect_mid_stream]": [ - "quota_management.spend_tracking.scripted_wire.client_disconnect" - ], - "tests/integration/mcp/test_mcp_lifecycle.py::test_health_intersects_route_restricted_key_grants_in_both_management_modes": [ - "other.mcp.health.restricted_keys_intersect_grants_in_both_modes" - ], - "tests/integration/mcp/test_mcp_lifecycle.py::test_warm_credential_removal_rejects_without_upstream_traffic": [ - "other.mcp.credentials.warm_removal_fails_closed_without_upstream_traffic" - ], - "tests/integration/observability/test_guardrail_effects.py::test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls": [ - "other.mcp.guardrails.request_selection_blocks_resolved_tool_without_execution" - ], - "tests/integration/mcp/test_oauth_configuration.py::test_same_url_oauth_credentials_and_revocation_are_isolated_by_user_and_server[revoke]": [ - "other.mcp.oauth.same_url_credentials_are_isolated_by_user_and_server" - ], - "tests/integration/mcp/test_oauth_configuration.py::test_same_url_oauth_credentials_and_revocation_are_isolated_by_user_and_server[expire]": [ - "other.mcp.oauth.same_url_credentials_are_isolated_by_user_and_server" - ], - "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[anonymous]": [ - "other.mcp.permissions.same_url_servers_enforce_discovery_and_execution" - ], - "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[bearer]": [ - "other.mcp.permissions.same_url_servers_enforce_discovery_and_execution" - ], - "tests/integration/management/test_budget_updates.py::test_shortening_budget_duration_moves_reset_at_onto_the_new_schedule": [ - "mgmt.budget.update.duration_change_recomputes_reset_at" - ], - "tests/integration/management/test_organization_budget_clear.py::test_patch_organization_update_with_null_tpm_limit_clears_it_and_keeps_sibling_limits": [ - "mgmt.organization.update.null_clears_budget_limit" - ], - "tests/integration/management/test_team_budget_duration_defaults.py::test_team_new_explicit_null_budget_duration_is_not_replaced_by_default": [ - "mgmt.team.new.explicit_null_budget_duration_overrides_default" - ], - "tests/integration/management/test_team_member_budget_cache.py::test_team_member_default_budget_lands_in_redis_after_first_member_call": [ - "mgmt.team_member_budget.default_budget_is_cached_in_redis_as_json" - ], - "tests/integration/observability/test_callback_delivery.py::test_streamed_responses_success_callback_carries_provider_apim_request_id": [ - "other.observability.callbacks.streamed_responses_events_carry_provider_response_headers" - ], - "tests/integration/observability/test_guardrail_effects.py::test_bedrock_passthrough_converse_guardrail_ignores_denied_term_in_tool_definition": [ - "other.observability.guardrails.bedrock_passthrough_converse_scans_only_caller_content" - ], - "tests/integration/pricing/test_configured_prices.py::test_cost_estimate_reports_configured_prices_for_model_absent_from_cost_map": [ - "quota_management.cost_estimate.configured_price.reported_for_model_absent_from_cost_map" - ], - "tests/integration/pricing/test_configured_prices.py::test_saving_echoed_model_info_does_not_freeze_cost_map_price_into_deployment": [ - "pricing.model_update.echoed_cost_map_price_is_not_persisted_as_override" - ], - "tests/integration/pricing/test_databricks_cache_pricing.py::test_databricks_cached_prompt_tokens_bill_at_cache_rates_not_input_rate": [ - "pricing.databricks.cached_prompt_tokens_bill_at_cache_rates" - ], - "tests/integration/pricing/test_ocr_page_pricing.py::test_ocr_annotation_pages_are_billed_at_annotation_cost_per_page": [ - "pricing.ocr.annotation_pages_billed_at_annotation_rate" - ], - "tests/integration/pricing/test_service_tier_pricing.py::test_ultrafast_service_tier_bills_ultrafast_rates_and_keeps_pricing_off_the_wire": [ - "quota_management.spend_tracking.service_tier_pricing.ultrafast_bills_ultrafast_rates" - ], - "tests/integration/spend/test_batch_completion_accounting.py::test_completed_batch_spend_row_records_reasoning_tokens_and_error_file_failures": [ - "quota_management.spend_tracking.batch_costs.reasoning_tokens_and_error_file_failures_recorded" - ], - "tests/integration/spend/test_batch_observability.py::test_batch_retrieval_row_sums_reasoning_tokens_and_counts_output_and_error_file_failures": [ - "spend.batches.retrieval_row_aggregates_reasoning_tokens_and_per_request_counts" - ], - "tests/integration/spend/test_batch_poll_starvation.py::test_batches_gone_at_provider_do_not_starve_a_newer_batch_out_of_cost_polling": [ - "quota_management.spend_tracking.batch_costs.uncostable_rows_retire_so_newer_batches_are_costed" - ], - "tests/integration/spend/test_cache_and_quota.py::test_in_flight_count_tokens_does_not_reserve_key_budget_away_from_a_completion": [ - "quota_management.budget.key.in_flight_count_tokens_reserves_nothing_so_completion_reaches_provider" - ], - "tests/integration/spend/test_cache_and_quota.py::test_repeated_count_tokens_on_budgeted_key_does_not_reserve_budget_or_block_later_completion": [ - "quota_management.budget.key.count_tokens_reserves_nothing_so_completion_within_budget_succeeds" - ], - "tests/integration/spend/test_daily_rollup_retry.py::test_failed_daily_user_rollup_commit_is_retried_so_spend_report_and_daily_activity_agree": [ - "spend.daily_rollup.failed_user_commit_is_retried_until_report_and_daily_activity_agree" - ], - "tests/integration/spend/test_disconnected_bedrock_messages_stream_billing.py::test_client_disconnect_mid_bedrock_messages_stream_still_bills_terminal_usage": [ - "spend.anthropic_messages_stream.client_disconnect_bills_terminal_bedrock_usage" - ], - "tests/integration/spend/test_failed_dispatch_tokens.py::test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_row": [ - "spend.failed_dispatch.failure_row_records_estimated_input_tokens" - ], - "tests/integration/spend/test_model_router_selected_model.py::test_model_router_alias_without_router_in_name_keeps_selected_model_in_response_and_spend_log": [ - "spend.model_router.selected_model_is_returned_and_persisted_for_plain_alias" - ], - "tests/integration/spend/test_org_budget_cli_session_token.py::test_cli_session_token_without_org_id_charges_and_caps_the_team_organization": [ - "quota_management.organization_budget.cli_session_token_without_org_id_charges_team_organization" - ], - "tests/integration/spend/test_passthrough_budget_reservation.py::test_repeated_gemini_passthrough_calls_stay_served_while_key_spend_is_below_max_budget": [ - "spend.budget_reservation.gemini_passthrough_success_releases_reservation_from_spend_counter" - ], - "tests/integration/spend/test_team_daily_activity_aggregated.py::test_aggregated_team_activity_reports_the_whole_range_team_spend_in_one_page": [ - "quota_management.spend_tracking.team_daily_activity_aggregated_reports_whole_range_team_spend" - ], - "tests/integration/spend/test_team_member_spend.py::test_member_added_without_any_budget_is_charged_on_its_membership_row": [ - "spend.team_member.member_without_budget_gets_membership_row_and_spend" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_is_listed_beside_db_store_and_survives_listing": [ - "mgmt.vector_store.list.keeps_config_store_beside_db_stores" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_refuses_new_update_and_delete": [ - "mgmt.vector_store.write.config_store_is_read_only" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_db_store_lifecycle_is_unchanged_beside_config_store": [ - "mgmt.vector_store.write.db_store_lifecycle_unchanged_beside_config_store" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_chat_with_config_store_searches_upstream_and_injects_context_after_listing": [ - "other.vector_store.chat.config_store_search_reaches_upstream_after_listing" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_passthrough_search_on_config_store_uses_yaml_credentials_after_listing": [ - "other.vector_store.search.config_store_passthrough_uses_yaml_credentials_after_listing" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_non_admin_key_access_to_config_store_follows_grants_after_admin_listing": [ - "authz.vector_store.list.non_admin_key_access_to_config_store_follows_grants" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_peer_process_keeps_config_store_and_sees_db_store_created_elsewhere": [ - "mgmt.vector_store.list.peer_process_keeps_config_store_and_sees_db_store" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_concurrent_burst_keeps_config_store_and_refuses_every_config_write": [ - "mgmt.vector_store.chaos.concurrent_burst_keeps_config_store_across_workers" - ], - "tests/integration/management/test_vector_store_config_ownership.py::test_redis_outage_keeps_config_store_served_and_recovers": [ - "mgmt.vector_store.chaos.redis_outage_keeps_config_store_and_recovers" - ], - "tests/integration/providers/test_anthropic_advisor_wire.py::test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated": [ - "providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment" - ], - "tests/integration/providers/test_azure_ai_chat_wire.py::test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages": [ - "providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry" - ], - "tests/integration/providers/test_azure_ai_flux2_image_wire.py::test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro": [ - "other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body" - ], - "tests/integration/providers/test_azure_ai_rerank_auth_wire.py::test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider": [ - "other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider" - ], - "tests/integration/providers/test_bedrock_auth_wire.py::test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature": [ - "providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization" - ], - "tests/integration/providers/test_bedrock_batch_files_wire.py::test_completions_and_responses_batch_records_upload_as_anthropic_user_messages": [ - "other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages" - ], - "tests/integration/providers/test_bedrock_claude_thinking_wire.py::test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens": [ - "other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking" - ], - "tests/integration/providers/test_bedrock_converse_config_blocks_wire.py::test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config": [ - "other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level" - ], - "tests/integration/providers/test_bedrock_embedding_wire.py::test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions": [ - "other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format" - ], - "tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py::test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort": [ - "providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort" - ], - "tests/integration/providers/test_bedrock_invoke_tool_search_wire.py::test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field": [ - "providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support" - ], - "tests/integration/providers/test_bedrock_mantle_codex_input_wire.py::test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items": [ - "other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types" - ], - "tests/integration/providers/test_bedrock_mantle_responses_wire.py::test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle": [ - "providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items" - ], - "tests/integration/providers/test_bedrock_mantle_wire.py::test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long": [ - "other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long" - ], - "tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py::test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers": [ - "providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text" - ], - "tests/integration/providers/test_bedrock_passthrough_stream_wire.py::test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type": [ - "other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type" - ], - "tests/integration/providers/test_bedrock_rerank_wire.py::test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature": [ - "providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned" - ], - "tests/integration/providers/test_bedrock_thinking_tokens_wire.py::test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens": [ - "other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text" - ], - "tests/integration/providers/test_dashscope_chat_wire.py::test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider": [ - "other.provider_wire.dashscope.reasoning_effort_reaches_provider" - ], - "tests/integration/providers/test_databricks_chat_wire.py::test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log": [ - "other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log" - ], - "tests/integration/providers/test_databricks_oauth_wire.py::test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin": [ - "other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base" - ], - "tests/integration/providers/test_deepseek_vision_wire.py::test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text": [ - "other.provider_wire.deepseek.vision_image_content_list_reaches_provider" - ], - "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_chat_sends_router_resource_not_models_path": [ - "other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name" - ], - "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path": [ - "other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name" - ], - "tests/integration/providers/test_openai_chat_wire.py::test_openai_chat_tool_choice_without_tools_is_not_forwarded": [ - "providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire" - ], - "tests/integration/providers/test_openai_image_edit_wire.py::test_openai_compatible_image_edit_forwards_seed_form_field_to_backend": [ - "other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields" - ], - "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out": [ - "other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length" - ], - "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[reasoning_effort_none]": [ - "other.provider_wire.tencent.thinking_reaches_provider_in_request_body" - ], - "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[thinking_enabled]": [ - "other.provider_wire.tencent.thinking_reaches_provider_in_request_body" - ], - "tests/integration/providers/test_websearch_interception_wire.py::test_capped_websearch_interception_loop_ends_turn_instead_of_exposing_internal_tool_use": [ - "other.provider_wire.anthropic.websearch_interception_capped_loop_ends_turn_without_internal_tool_use" - ], - "tests/integration/providers/test_websearch_interception_wire.py::test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks": [ - "other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results" - ], - "tests/integration/providers/test_xai_web_search_wire.py::test_xai_chat_web_search_is_sent_to_responses_with_instructions_and_nested_filters": [ - "other.provider_wire.xai.chat_web_search_reaches_responses_with_instructions_and_filters" - ], - "tests/integration/routing/test_priority_rate_limit_headers.py::test_non_streaming_v1_messages_success_carries_v3_priority_rate_limit_headers": [ - "other.routing.priority_rate_limits.v1_messages_success_exposes_v3_priority_headers" - ], - "tests/integration/routing/test_stale_cost_map_boot.py::test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_reload": [ - "other.routing.cost_map.config_deployment_dropped_by_stale_boot_map_is_restored_after_reload" - ], - "tests/integration/streaming/test_stream_contracts.py::test_messages_stream_completes_through_trailing_empty_choices_usage_chunk": [ - "other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream" - ], - "tests/integration/streaming/test_stream_contracts.py::test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost": [ - "other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost" - ], - "tests/integration/streaming/test_stream_contracts.py::test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bills_the_fallback": [ - "other.streaming.fallback.empty_leading_chunk_then_disconnect_streams_fallback_with_usage_and_spend" - ], - "tests/integration/streaming/test_stream_contracts.py::test_responses_stream_completes_through_empty_choices_metadata_and_usage_chunks": [ - "other.streaming.responses_bridge.empty_choices_chunks_complete_stream" - ], - "tests/integration/streaming/test_stream_parallel_slot_release.py::test_failing_stream_logging_callback_does_not_leak_max_parallel_requests_slot": [ - "streaming.max_parallel_requests.slot_released_when_stream_logging_callback_fails" - ], - "tests/integration/streaming/test_ttft_keepalive.py::test_stream_emits_sse_ping_comments_before_the_first_data_frame_while_upstream_is_silent": [ - "streaming.keepalive.sse_pings_fill_silent_time_to_first_token" - ], - "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": [ - "other.observability.callbacks.raising_success_deployment_hook_keeps_response" - ], - "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[embeddings]": [ - "other.observability.callbacks.raising_success_deployment_hook_keeps_response" - ], - "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[responses]": [ - "other.observability.callbacks.raising_success_deployment_hook_keeps_response" - ], - "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[videos]": [ - "other.observability.callbacks.raising_success_deployment_hook_keeps_response" - ], - "tests/integration/management/test_user_updates_wedged_coordination_redis.py::test_user_budget_updates_return_promptly_while_coordination_redis_is_wedged": [ - "mgmt.user.update.budget_change_returns_promptly_with_wedged_coordination_redis", - "mgmt.user.bulk_update.budget_change_returns_promptly_with_wedged_coordination_redis", - "mgmt.customer.update.budget_change_returns_promptly_with_wedged_coordination_redis", - "mgmt.key.reset_spend.returns_promptly_with_wedged_coordination_redis", - "mgmt.auth_cache_invalidation.publish_parked_by_short_redis_wedge_lands_after_recovery", - "mgmt.auth_cache_invalidation.burst_with_worker_kill_keeps_serving_while_redis_wedged" - ] - }, - "browser": { - "tests/e2e/ui/tests/integrationCritical/projectDetachment.spec.ts::project creation and explicit detachment preserve saved scope and restore serving": [ - "mgmt.key.ui.project_create_clear_preserves_serving_scope" - ] - } -} +{"groups": {"management": ["management", "authorization", "configuration"], "accounting": ["pricing", "spend"], "database": ["database"], "providers": ["providers", "routing", "streaming"], "extensions": ["mcp", "observability", "compatibility"], "sdk": ["sdk"], "cost": ["cost_calculation"]}, "tests": {"tests/integration/management/test_key_updates.py::test_update_preserves_independent_fields_and_serving": ["mgmt.key.update.preserves_independent_fields"], "tests/integration/pricing/test_configured_prices.py::test_custom_price_is_reported_and_charged": ["quota_management.spend_tracking.custom_price.matches_input_rates"], "tests/integration/providers/test_request_boundary.py::test_internal_request_state_does_not_reach_provider": ["other.provider_wire.internal_parameters_filtered"], "tests/integration/pricing/test_configured_prices.py::test_default_prices_survive_nullable_sibling_and_reload": ["quota_management.spend_tracking.default_prices.survive_nullable_sibling_reload"], "tests/integration/providers/test_request_boundary.py::test_upstream_rejects_corruption_and_accepts_supported_metadata": ["other.provider_wire.validator_rejects_corruption"], "tests/integration/pricing/test_configured_prices.py::test_loaded_router_preserves_cached_defaults_during_real_requests": ["quota_management.spend_tracking.default_prices.loaded_router_preserves_cached_defaults"], "tests/integration/management/test_partial_update_sequences.py::test_generated_partial_updates_preserve_persisted_and_effective_state": ["mgmt.key.update.generated_sequences_preserve_state"], "tests/integration/management/test_partial_update_sequences.py::test_zero_false_and_empty_values_are_not_treated_as_omission": ["mgmt.key.update.false_zero_and_empty_values_affect_serving"], "tests/integration/management/test_partial_update_sequences.py::test_project_omission_clear_and_invalid_update_have_distinct_effects": ["mgmt.key.update.project_clear_preserves_scope", "mgmt.key.update.invalid_batch_is_atomic"], "tests/integration/authorization/test_warmed_policy.py::test_generated_policy_changes_reach_both_warmed_workers": ["mgmt.key.update.two_workers_enforce_warmed_policy"], "tests/integration/authorization/test_warmed_policy.py::test_scim_deactivation_blocks_null_and_false_keys_but_preserves_other_owners": ["mgmt.user.scim.deactivation_includes_nullable_blocked_keys"], "tests/integration/authorization/test_warmed_policy.py::test_warmed_team_role_demotion_prevents_later_management_writes": ["mgmt.team.member_update.demoted_role_cannot_write"], "tests/integration/configuration/test_effective_settings.py::test_model_block_changes_actual_route_and_leaves_other_route_working": ["mgmt.model.block.changes_serving_and_preserves_control"], "tests/integration/configuration/test_effective_settings.py::test_saved_retry_setting_controls_real_attempts_and_restores": ["mgmt.router_settings.update.changes_observed_attempt_count"], "tests/integration/configuration/test_effective_settings.py::test_credential_value_update_and_model_reload_reach_provider": ["mgmt.credential.update.saved_value_reaches_wire"], "tests/integration/management/test_partial_update_sequences.py::test_denied_key_update_preserves_saved_grants_and_serving": ["mgmt.key.update.denied_request_preserves_effective_state"], "tests/integration/authorization/test_warmed_policy.py::test_expiry_and_explicit_clear_reach_both_warmed_workers": ["mgmt.key.update.expiry_changes_reach_warmed_workers"], "tests/integration/database/test_partition_transactions.py::test_real_partition_ddl_survives_witnessed_lock_and_is_idempotent": ["other.database.partitions.lock_wait_outlives_transaction_default", "other.database.partitions.repeat_preserves_rows"], "tests/integration/database/test_reader_writer_regeneration.py::test_key_regeneration_uses_writer_with_a_real_readonly_reader": ["other.database.regeneration.writer_updates_dependent_grants"], "tests/integration/pricing/test_price_precedence.py::test_generated_zero_null_and_omitted_prices_follow_independent_arithmetic": ["quota_management.spend_tracking.price_precedence.zero_and_default_rates"], "tests/integration/pricing/test_price_precedence.py::test_same_upstream_aliases_keep_distinct_prices_after_reload": ["quota_management.spend_tracking.alias_prices.remain_independent_on_reload"], "tests/integration/pricing/test_off_peak_pricing.py::test_open_off_peak_window_bills_off_peak_rates": ["quota_management.spend_tracking.off_peak_pricing.open_window_bills_off_peak_rates"], "tests/integration/pricing/test_off_peak_pricing.py::test_closed_off_peak_window_bills_standard_rates": ["quota_management.spend_tracking.off_peak_pricing.closed_window_bills_standard_rates"], "tests/integration/spend/test_cache_and_quota.py::test_generated_cache_sequences_preserve_content_usage_and_zero_hit_cost": ["quota_management.response_cache.generated_sequences_preserve_content_and_accounting"], "tests/integration/spend/test_cache_and_quota.py::test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores": ["quota_management.budget.key.boundary_blocks_before_provider_and_reset_restores"], "tests/integration/spend/test_cache_and_quota.py::test_different_system_messages_do_not_share_a_cached_response": ["quota_management.response_cache.system_messages_partition_cache_identity"], "tests/integration/database/test_transaction_atomicity.py::test_access_group_second_key_constraint_failure_rolls_back_all_writes": ["other.database.access_group.failed_second_write_rolls_back_first"], "tests/integration/spend/test_cache_and_quota.py::test_repeated_hits_keep_response_identity_and_create_distinct_zero_cost_rows": ["quota_management.response_cache.repeated_hits_preserve_identity_and_single_charge"], "tests/integration/providers/test_s3_wire.py::test_sigv4_verifier_matches_published_put_and_rejects_corruption": ["other.provider_wire.s3.verifier_known_answer_and_negative_controls"], "tests/integration/providers/test_s3_wire.py::test_s3_sync_and_async_uploads_pass_independent_wire_verification": ["other.provider_wire.s3.sync_async_reserved_keys_are_signed_and_accepted"], "tests/integration/providers/test_bedrock_auth_wire.py::test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials": ["other.provider_wire.bedrock.bearer_sdk_skips_credential_chain"], "tests/integration/providers/test_bedrock_auth_wire.py::test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload": ["other.provider_wire.bedrock.bearer_db_yaml_survives_reload"], "tests/integration/streaming/test_stream_contracts.py::test_generated_tcp_partitions_preserve_unicode_text_identity_and_final_usage": ["other.streaming.byte_partitions.preserve_text_identity_and_usage"], "tests/integration/streaming/test_stream_contracts.py::test_fragmented_tool_names_and_arguments_keep_each_call_identity": ["other.streaming.tools.fragmented_calls_keep_independent_arguments"], "tests/integration/streaming/test_stream_contracts.py::test_proxy_stream_usage_visibility_keeps_exact_persisted_charge": ["other.streaming.usage.client_visibility_preserves_persisted_accounting"], "tests/integration/streaming/test_stream_contracts.py::test_truncated_http_stream_is_an_error_and_next_stream_succeeds": ["other.streaming.failure.truncated_transport_raises_and_control_recovers"], "tests/integration/streaming/test_stream_contracts.py::test_client_cancellation_releases_the_actual_provider_connection": ["other.streaming.cancellation.closes_actual_provider_connection"], "tests/integration/routing/test_observed_routing.py::test_retry_counts_and_public_errors_match_actual_provider_attempts": ["other.routing.retries.several_attempts_reach_success_without_hidden_retries", "other.routing.errors.nonretryable_and_exhausted_failures_remain_errors"], "tests/integration/routing/test_observed_routing.py::test_loaded_fallback_selects_expected_deployment_and_keeps_response_identity": ["other.routing.fallback.loaded_configuration_selects_only_permitted_target"], "tests/integration/routing/test_observed_routing.py::test_saved_deployment_target_update_changes_wire_and_preserves_control": ["other.routing.alias_update.persisted_target_changes_only_selected_route"], "tests/integration/providers/test_bedrock_role_configuration.py::test_role_reference_from_db_and_yaml_reaches_real_sts_http_and_bedrock": ["other.provider_wire.bedrock.db_yaml_role_reference_reaches_sts_and_signed_request"], "tests/integration/routing/test_redis_recovery.py::test_owned_redis_outage_recovers_requests_and_real_response_cache": ["other.routing.redis.owned_outage_recovers_serving_and_response_cache"], "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[type_word]": ["other.provider_wire.anthropic.bare_string_content_item_is_client_error"], "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[plain]": ["other.provider_wire.anthropic.bare_string_content_item_is_client_error"], "tests/integration/providers/test_anthropic_wire.py::test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contracts": ["other.provider_wire.anthropic.tool_history_system_cache_and_internal_fields", "quota_management.spend_tracking.cache_tokens.disjoint_classes_use_explicit_rates"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_create_status_and_content_follow_queue_wire_contract": ["other.provider_wire.fal_ai.video_queue_create_status_and_content_download"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_failed_result_reports_failed_status_and_fal_error": ["other.provider_wire.fal_ai.video_failed_result_surfaces_fal_error"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_auto_duration_omits_duration_and_queues": ["other.provider_wire.fal_ai.h3_auto_duration_omits_duration_and_queues"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_oversized_size_uses_top_resolution_tier_and_queues": ["other.provider_wire.fal_ai.h3_oversized_size_uses_top_resolution_tier"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": ["other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_prices_non_canonical_size_from_nearest_row": ["other.provider_wire.fal_ai.gpt_image_generation_noncanonical_size_uses_nearest_keyed_row"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_sdk_response_honors_dump_options": ["other.provider_wire.fal_ai.sdk_image_response_dump_options"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": ["other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing"], "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_charges_and_polls_pass_through_free": ["other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not"], "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_prices_string_resolution_like_the_integer": ["other.provider_wire.fal_ai.passthrough_queue_submit_prices_string_resolution_like_integer"], "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_to_catalog_key_the_pricer_cannot_price_is_rejected_not_forwarded": ["other.provider_wire.fal_ai.passthrough_queue_submit_rejects_unpriceable_catalog_key"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": ["other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_lora_depth_edit_sends_single_image_url_and_charges_flat_row": ["other.provider_wire.fal_ai.flux_lora_depth_edit_single_image_url_and_flat_pricing"], "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_sends_prompt_image_and_reasoning": ["other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing"], "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_rejects_non_string_reasoning_effort_before_the_wire": ["other.provider_wire.fal_ai.chat_non_string_reasoning_effort_rejected_before_wire"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_without_deployment_api_base_uses_global_api_base": ["other.provider_wire.fal_ai.global_api_base_routes_image_generation"], "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-pro]": ["other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing"], "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-flash]": ["other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing"], "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas": ["other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas"], "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_tool_call_is_forwarded_and_returned": ["other.provider_wire.xiaomi_mimo.tool_call_survives_translation"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_video_create_uses_canonical_body_and_status_path": ["other.provider_wire.fal_ai.video_queue_create_status_and_content_download"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_carries_the_deployment_extra_headers": ["other.provider_wire.fal_ai.video_result_probe_forwards_extra_headers"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_reuses_the_ssl_verify_false_client": ["other.provider_wire.fal_ai.video_result_probe_honors_ssl_verify"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_provider_hanging_up_on_the_result_probe_keeps_the_completed_status": ["other.provider_wire.fal_ai.video_result_probe_hangup_stays_completed"], "tests/integration/mcp/test_mcp_lifecycle.py::test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit": ["mcp.call_tool.saved_headers.reach_actual_transport"], "tests/integration/mcp/test_mcp_lifecycle.py::test_tool_error_remains_error_and_healthy_sibling_returns_value": ["mcp.call_tool.errors.tool_failure_is_not_success"], "tests/integration/mcp/test_mcp_lifecycle.py::test_generated_mcp_edits_preserve_actual_headers_and_tool_results": ["other.mcp.lifecycle.generated_save_reload_preserves_effective_headers"], "tests/integration/observability/test_callback_delivery.py::test_concurrent_success_and_failure_join_callbacks_and_rows_without_credentials": ["other.observability.callbacks.credentials_stay_out_of_event_bodies", "other.observability.callbacks.concurrent_results_join_complete_events_and_rows"], "tests/integration/observability/test_otel_text_completion_choices.py::test_otel_weave_output_keeps_text_completion_provider_fields_beside_the_synthesized_message": ["other.observability.otel.text_completion_choices_keep_provider_fields"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys": ["other.observability.langfuse_otel.request_metadata_on_langfuse_keys"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse": ["other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream": ["other.observability.langfuse_otel.request_metadata_openai_sdk_stream"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async": ["other.observability.langfuse_otel.request_metadata_openai_sdk_async"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages": ["other.observability.langfuse_otel.request_metadata_anthropic_sdk_messages"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream": ["other.observability.langfuse_otel.request_metadata_anthropic_sdk_stream"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_api": ["other.observability.langfuse_otel.request_metadata_responses_api"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream": ["other.observability.langfuse_otel.request_metadata_responses_stream_sse"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_caller_trace_metadata_survives_flattened_identity": ["other.observability.langfuse_otel.caller_trace_metadata_preserved_with_flattened_identity"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_failure_span_carries_request_metadata": ["other.observability.langfuse_otel.failure_span_carries_request_metadata"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_emits_no_team_identity": ["other.observability.langfuse_otel.teamless_key_omits_team_identity"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_keeps_caller_trace_metadata": ["other.observability.langfuse_otel.teamless_key_keeps_caller_trace_metadata"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_values": ["other.observability.langfuse_otel.spend_logs_metadata_roundtrips_nonstring_values"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_unauthenticated_request_leaks_no_identity": ["other.observability.langfuse_otel.unauthenticated_request_leaks_no_identity"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_repeated_identical_requests_emit_one_span_each": ["other.observability.langfuse_otel.repeated_requests_each_emit_one_span"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once": ["other.observability.langfuse_otel.sink_outage_burst_lands_each_id_once"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports": ["other.observability.langfuse_otel.worker_kill_burst_keeps_serving"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_slow_sink_does_not_deadlock_exports": ["other.observability.langfuse_otel.slow_sink_does_not_deadlock_exports"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_v2_mapper_keeps_existing_trace_metadata_keys": ["other.observability.langfuse_otel.v2_mapper_unchanged"], "tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": ["other.observability.guardrails.rewrite_reaches_correct_anthropic_positions"], "tests/integration/compatibility/test_a2a_wire_versions.py::test_a2a_versions_and_legacy_casing_preserve_real_wire_and_response": ["other.compatibility.a2a.supported_versions_preserve_literal_envelopes"], "tests/integration/compatibility/test_persisted_toolsets.py::test_existing_toolset_format_loads_before_start_and_keeps_sibling_denied": ["other.compatibility.mcp.persisted_tool_names_survive_candidate_startup"], "tests/integration/mcp/test_oauth_configuration.py::test_partial_discovery_and_unrelated_edit_keep_actual_authorization_destination": ["other.mcp.oauth.discovery_cannot_erase_configured_authorization_endpoint"], "tests/integration/observability/test_guardrail_effects.py::test_guardrail_denial_prevents_provider_and_preserves_allowed_control": ["other.observability.guardrails.denial_prevents_provider_with_allowed_control"], "tests/integration/mcp/test_mcp_protocol_errors.py::test_jsonrpc_error_and_malformed_tool_result_remain_errors": ["other.mcp.errors.protocol_and_malformed_results_cannot_be_empty_success"], "tests/integration/compatibility/test_openai_consumer.py::test_retained_openai_clients_parse_real_proxy_tool_and_usage_responses": ["other.compatibility.openai.retained_client_parses_tools_and_usage"], "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": ["quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals"], "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_written_by_the_final_flush": ["quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch"], "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_row_lock_is_written_exactly_once": ["quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch"], "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": ["quota_management.spend_tracking.spend_calculate.rejects_unpriced_model"], "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini-live-2.5-flash-preview-native-audio-09-2025]": ["quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"], "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini/gemini-live-2.5-flash-preview-native-audio-09-2025]": ["quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"], "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": ["mgmt.key.update.project_detach_denied_to_restricted_actor"], "tests/integration/management/test_partial_update_sequences.py::test_cross_tenant_actor_cannot_read_update_or_detach_project_key": ["mgmt.key.info.cross_tenant_key_is_denied", "mgmt.key.update.cross_tenant_key_is_denied", "mgmt.key.update.cross_tenant_project_detach_is_denied"], "tests/integration/management/test_project_lifecycle.py::test_project_new_persists_real_state": ["mgmt.project.new.real_route_persists"], "tests/integration/management/test_project_lifecycle.py::test_project_update_persists_real_state": ["mgmt.project.update.real_route_persists"], "tests/integration/management/test_project_lifecycle.py::test_project_delete_with_attached_key_refuses_and_preserves_state": ["mgmt.project.delete.attached_key_refusal_preserves_state"], "tests/integration/sdk/test_http2_wire.py::test_async_handler_negotiates_http2_only_when_enabled": ["other.sdk_wire.http2.async_handler_negotiates_h2_only_when_enabled"], "tests/integration/sdk/test_http2_wire.py::test_sync_handler_negotiates_http2_only_when_enabled": ["other.sdk_wire.http2.sync_handler_negotiates_h2_only_when_enabled"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-halved_rates_when_map_has_no_batch_keys]": ["quota_management.spend_tracking.batch_costs.fallback_rates"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-cached_input_halved]": ["quota_management.spend_tracking.batch_costs.cached_input"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.4-batch-explicit_batch_rates_bill_cached_at_batch_input_rate]": ["quota_management.spend_tracking.batch_costs.explicit_rates"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-all_requests_failed_zero_spend]": ["quota_management.spend_tracking.batch_costs.failed_requests"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-single_turn_text_audio_cached]": ["quota_management.spend_tracking.realtime_costs.single_turn"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-two_turns_summed_into_one_row]": ["quota_management.spend_tracking.realtime_costs.multiple_turns"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-priced_from_session_created_model]": ["quota_management.spend_tracking.realtime_costs.session_model"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-session_without_turns_zero_spend]": ["quota_management.spend_tracking.realtime_costs.session_without_turns"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_native_json]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_429_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-anthropic_us_inference]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_write_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_fast_mode]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_us_inference]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_write_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-anthropic_us_inference]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-fallback_cache_read_at_half_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-image_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-google_maps_grounding]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-fallback_video_tokens_at_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-video_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-web_search_per_prompt]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-google_maps_grounding]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_reasoning_at_output_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_image_tokens_at_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-image_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-video_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-google_maps_grounding]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-image_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-video_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-web_search_per_prompt]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-google_maps_grounding]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-file_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_incomplete]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_incomplete]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_unvalidated]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_unvalidated]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-file_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_incomplete]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_incomplete]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_unvalidated]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_unvalidated]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_read_at_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_write_at_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_write_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-next-transcriptions-per-second]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-verbose-next-transcriptions-duration]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-4o-transcribe-next-transcriptions-tokens]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[nova-next-transcriptions-per-second]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-whisper-next-transcriptions-deployment]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-speech-per-character]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-hd-speech-per-character]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-tts-next-speech-deployment]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-standard]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-hd]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-wide]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-two]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[imagen-next-images-one]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[amazon-nova-canvas-next-images-one]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-edit]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embeddings-4-large-deployment]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-embeddings-v4]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-rerank-v4]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-embeddings-titan-v2]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-embeddings-v5]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-one]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-three]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-total-tokens-fallback]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks-embeddings-v1]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-embeddings-002]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-list]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-single]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-basic]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-n-best]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-stream-usage]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-3-large-dimensions]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-batch]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-single]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-token-array]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-completions-v1]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-embeddings-v1]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[vertex-embeddings-text-006]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream_cache_read]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_incomplete]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_previous_response_id]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-responses_file_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_web_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream_cache_read]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-messages_input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-messages_input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-messages_cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-passthrough-generate_content_priced_via_gemini_key]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-passthrough-stream_generate_content_priced_via_vertex_key]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-passthrough-messages]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-passthrough-messages_cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse_stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_boundary_stays_lower_tier]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_second_tier]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_above_top_range]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_below_128k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_above_128k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_creation_1h_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-provider_reported_cost]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-token_priced]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-no_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-prompt_cache_hit]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-no_cache_fields_bills_zero_cache]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-reasoning_folded_into_completion]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-live_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-provider_reported_cost]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-invoke-haiku-json]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-invoke-haiku-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-profile-base-model]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-eu-regional-key]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-apac-bare-fallback]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-nova-2-pro]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-mistral-large-3-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-ai-gpt-5.4-mini-latest]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-ai-gpt-5.4-mini-latest-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-pinned-gpt-5.4-mini-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[groq-qwen-3.8-json]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[groq-qwen-3.8-stream_x_groq_recount]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-command-a-v2-tokens]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[mistral-medium-2604-json]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openai-deployment-pricing-override]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_400_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_401_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_stream_request_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_upstream_500_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_upstream_500_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-fallback_billed_to_answering_deployment]": ["quota_management.spend_tracking.routing.fallback_billing"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-n_2_choices]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-finish_reason_length]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_usage_in_empty_choices_chunk]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_usage_in_last_delta_chunk]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-unknown_model_response_model_unknown]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-unknown_model_response_model_known]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-chat_request_to_embedding_entry]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-client_disconnect_mid_stream]": ["quota_management.spend_tracking.scripted_wire.client_disconnect"], "tests/integration/mcp/test_mcp_lifecycle.py::test_health_intersects_route_restricted_key_grants_in_both_management_modes": ["other.mcp.health.restricted_keys_intersect_grants_in_both_modes"], "tests/integration/mcp/test_mcp_lifecycle.py::test_warm_credential_removal_rejects_without_upstream_traffic": ["other.mcp.credentials.warm_removal_fails_closed_without_upstream_traffic"], "tests/integration/observability/test_guardrail_effects.py::test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls": ["other.mcp.guardrails.request_selection_blocks_resolved_tool_without_execution"], "tests/integration/mcp/test_oauth_configuration.py::test_same_url_oauth_credentials_and_revocation_are_isolated_by_user_and_server[revoke]": ["other.mcp.oauth.same_url_credentials_are_isolated_by_user_and_server"], "tests/integration/mcp/test_oauth_configuration.py::test_same_url_oauth_credentials_and_revocation_are_isolated_by_user_and_server[expire]": ["other.mcp.oauth.same_url_credentials_are_isolated_by_user_and_server"], "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[anonymous]": ["other.mcp.permissions.same_url_servers_enforce_discovery_and_execution"], "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[bearer]": ["other.mcp.permissions.same_url_servers_enforce_discovery_and_execution"], "tests/integration/management/test_budget_updates.py::test_shortening_budget_duration_moves_reset_at_onto_the_new_schedule": ["mgmt.budget.update.duration_change_recomputes_reset_at"], "tests/integration/management/test_organization_budget_clear.py::test_patch_organization_update_with_null_tpm_limit_clears_it_and_keeps_sibling_limits": ["mgmt.organization.update.null_clears_budget_limit"], "tests/integration/management/test_team_budget_duration_defaults.py::test_team_new_explicit_null_budget_duration_is_not_replaced_by_default": ["mgmt.team.new.explicit_null_budget_duration_overrides_default"], "tests/integration/management/test_team_member_budget_cache.py::test_team_member_default_budget_lands_in_redis_after_first_member_call": ["mgmt.team_member_budget.default_budget_is_cached_in_redis_as_json"], "tests/integration/observability/test_callback_delivery.py::test_streamed_responses_success_callback_carries_provider_apim_request_id": ["other.observability.callbacks.streamed_responses_events_carry_provider_response_headers"], "tests/integration/observability/test_guardrail_effects.py::test_bedrock_passthrough_converse_guardrail_ignores_denied_term_in_tool_definition": ["other.observability.guardrails.bedrock_passthrough_converse_scans_only_caller_content"], "tests/integration/pricing/test_configured_prices.py::test_cost_estimate_reports_configured_prices_for_model_absent_from_cost_map": ["quota_management.cost_estimate.configured_price.reported_for_model_absent_from_cost_map"], "tests/integration/pricing/test_configured_prices.py::test_saving_echoed_model_info_does_not_freeze_cost_map_price_into_deployment": ["pricing.model_update.echoed_cost_map_price_is_not_persisted_as_override"], "tests/integration/pricing/test_databricks_cache_pricing.py::test_databricks_cached_prompt_tokens_bill_at_cache_rates_not_input_rate": ["pricing.databricks.cached_prompt_tokens_bill_at_cache_rates"], "tests/integration/pricing/test_ocr_page_pricing.py::test_ocr_annotation_pages_are_billed_at_annotation_cost_per_page": ["pricing.ocr.annotation_pages_billed_at_annotation_rate"], "tests/integration/pricing/test_service_tier_pricing.py::test_ultrafast_service_tier_bills_ultrafast_rates_and_keeps_pricing_off_the_wire": ["quota_management.spend_tracking.service_tier_pricing.ultrafast_bills_ultrafast_rates"], "tests/integration/spend/test_batch_completion_accounting.py::test_completed_batch_spend_row_records_reasoning_tokens_and_error_file_failures": ["quota_management.spend_tracking.batch_costs.reasoning_tokens_and_error_file_failures_recorded"], "tests/integration/spend/test_batch_observability.py::test_batch_retrieval_row_sums_reasoning_tokens_and_counts_output_and_error_file_failures": ["spend.batches.retrieval_row_aggregates_reasoning_tokens_and_per_request_counts"], "tests/integration/spend/test_batch_poll_starvation.py::test_batches_gone_at_provider_do_not_starve_a_newer_batch_out_of_cost_polling": ["quota_management.spend_tracking.batch_costs.uncostable_rows_retire_so_newer_batches_are_costed"], "tests/integration/spend/test_cache_and_quota.py::test_in_flight_count_tokens_does_not_reserve_key_budget_away_from_a_completion": ["quota_management.budget.key.in_flight_count_tokens_reserves_nothing_so_completion_reaches_provider"], "tests/integration/spend/test_cache_and_quota.py::test_repeated_count_tokens_on_budgeted_key_does_not_reserve_budget_or_block_later_completion": ["quota_management.budget.key.count_tokens_reserves_nothing_so_completion_within_budget_succeeds"], "tests/integration/spend/test_daily_rollup_retry.py::test_failed_daily_user_rollup_commit_is_retried_so_spend_report_and_daily_activity_agree": ["spend.daily_rollup.failed_user_commit_is_retried_until_report_and_daily_activity_agree"], "tests/integration/spend/test_disconnected_bedrock_messages_stream_billing.py::test_client_disconnect_mid_bedrock_messages_stream_still_bills_terminal_usage": ["spend.anthropic_messages_stream.client_disconnect_bills_terminal_bedrock_usage"], "tests/integration/spend/test_failed_dispatch_tokens.py::test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_row": ["spend.failed_dispatch.failure_row_records_estimated_input_tokens"], "tests/integration/spend/test_model_router_selected_model.py::test_model_router_alias_without_router_in_name_keeps_selected_model_in_response_and_spend_log": ["spend.model_router.selected_model_is_returned_and_persisted_for_plain_alias"], "tests/integration/spend/test_org_budget_cli_session_token.py::test_cli_session_token_without_org_id_charges_and_caps_the_team_organization": ["quota_management.organization_budget.cli_session_token_without_org_id_charges_team_organization"], "tests/integration/spend/test_passthrough_budget_reservation.py::test_repeated_gemini_passthrough_calls_stay_served_while_key_spend_is_below_max_budget": ["spend.budget_reservation.gemini_passthrough_success_releases_reservation_from_spend_counter"], "tests/integration/spend/test_team_daily_activity_aggregated.py::test_aggregated_team_activity_reports_the_whole_range_team_spend_in_one_page": ["quota_management.spend_tracking.team_daily_activity_aggregated_reports_whole_range_team_spend"], "tests/integration/spend/test_team_member_spend.py::test_member_added_without_any_budget_is_charged_on_its_membership_row": ["spend.team_member.member_without_budget_gets_membership_row_and_spend"], "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_is_listed_beside_db_store_and_survives_listing": ["mgmt.vector_store.list.keeps_config_store_beside_db_stores"], "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_refuses_new_update_and_delete": ["mgmt.vector_store.write.config_store_is_read_only"], "tests/integration/management/test_vector_store_config_ownership.py::test_db_store_lifecycle_is_unchanged_beside_config_store": ["mgmt.vector_store.write.db_store_lifecycle_unchanged_beside_config_store"], "tests/integration/management/test_vector_store_config_ownership.py::test_chat_with_config_store_searches_upstream_and_injects_context_after_listing": ["other.vector_store.chat.config_store_search_reaches_upstream_after_listing"], "tests/integration/management/test_vector_store_config_ownership.py::test_passthrough_search_on_config_store_uses_yaml_credentials_after_listing": ["other.vector_store.search.config_store_passthrough_uses_yaml_credentials_after_listing"], "tests/integration/management/test_vector_store_config_ownership.py::test_non_admin_key_access_to_config_store_follows_grants_after_admin_listing": ["authz.vector_store.list.non_admin_key_access_to_config_store_follows_grants"], "tests/integration/management/test_vector_store_config_ownership.py::test_peer_process_keeps_config_store_and_sees_db_store_created_elsewhere": ["mgmt.vector_store.list.peer_process_keeps_config_store_and_sees_db_store"], "tests/integration/management/test_vector_store_config_ownership.py::test_concurrent_burst_keeps_config_store_and_refuses_every_config_write": ["mgmt.vector_store.chaos.concurrent_burst_keeps_config_store_across_workers"], "tests/integration/management/test_vector_store_config_ownership.py::test_redis_outage_keeps_config_store_served_and_recovers": ["mgmt.vector_store.chaos.redis_outage_keeps_config_store_and_recovers"], "tests/integration/providers/test_anthropic_advisor_wire.py::test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated": ["providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment"], "tests/integration/providers/test_azure_ai_chat_wire.py::test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages": ["providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry"], "tests/integration/providers/test_azure_ai_flux2_image_wire.py::test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro": ["other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body"], "tests/integration/providers/test_azure_ai_rerank_auth_wire.py::test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider": ["other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider"], "tests/integration/providers/test_bedrock_auth_wire.py::test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature": ["providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization"], "tests/integration/providers/test_bedrock_batch_files_wire.py::test_completions_and_responses_batch_records_upload_as_anthropic_user_messages": ["other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages"], "tests/integration/providers/test_bedrock_claude_thinking_wire.py::test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens": ["other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking"], "tests/integration/providers/test_bedrock_converse_config_blocks_wire.py::test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config": ["other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level"], "tests/integration/providers/test_bedrock_embedding_wire.py::test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions": ["other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format"], "tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py::test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort": ["providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort"], "tests/integration/providers/test_bedrock_invoke_tool_search_wire.py::test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field": ["providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support"], "tests/integration/providers/test_bedrock_mantle_codex_input_wire.py::test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items": ["other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types"], "tests/integration/providers/test_bedrock_mantle_responses_wire.py::test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle": ["providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items"], "tests/integration/providers/test_bedrock_mantle_wire.py::test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long": ["other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long"], "tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py::test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers": ["providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text"], "tests/integration/providers/test_bedrock_passthrough_stream_wire.py::test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type": ["other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type"], "tests/integration/providers/test_bedrock_rerank_wire.py::test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature": ["providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned"], "tests/integration/providers/test_bedrock_thinking_tokens_wire.py::test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens": ["other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text"], "tests/integration/providers/test_dashscope_chat_wire.py::test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider": ["other.provider_wire.dashscope.reasoning_effort_reaches_provider"], "tests/integration/providers/test_databricks_chat_wire.py::test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log": ["other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log"], "tests/integration/providers/test_databricks_oauth_wire.py::test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin": ["other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base"], "tests/integration/providers/test_deepseek_vision_wire.py::test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text": ["other.provider_wire.deepseek.vision_image_content_list_reaches_provider"], "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_chat_sends_router_resource_not_models_path": ["other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name"], "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path": ["other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name"], "tests/integration/providers/test_openai_chat_wire.py::test_openai_chat_tool_choice_without_tools_is_not_forwarded": ["providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire"], "tests/integration/providers/test_openai_image_edit_wire.py::test_openai_compatible_image_edit_forwards_seed_form_field_to_backend": ["other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields"], "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out": ["other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length"], "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[reasoning_effort_none]": ["other.provider_wire.tencent.thinking_reaches_provider_in_request_body"], "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[thinking_enabled]": ["other.provider_wire.tencent.thinking_reaches_provider_in_request_body"], "tests/integration/providers/test_websearch_interception_wire.py::test_capped_websearch_interception_loop_ends_turn_instead_of_exposing_internal_tool_use": ["other.provider_wire.anthropic.websearch_interception_capped_loop_ends_turn_without_internal_tool_use"], "tests/integration/providers/test_websearch_interception_wire.py::test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks": ["other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results"], "tests/integration/providers/test_xai_web_search_wire.py::test_xai_chat_web_search_is_sent_to_responses_with_instructions_and_nested_filters": ["other.provider_wire.xai.chat_web_search_reaches_responses_with_instructions_and_filters"], "tests/integration/routing/test_priority_rate_limit_headers.py::test_non_streaming_v1_messages_success_carries_v3_priority_rate_limit_headers": ["other.routing.priority_rate_limits.v1_messages_success_exposes_v3_priority_headers"], "tests/integration/routing/test_stale_cost_map_boot.py::test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_reload": ["other.routing.cost_map.config_deployment_dropped_by_stale_boot_map_is_restored_after_reload"], "tests/integration/streaming/test_stream_contracts.py::test_messages_stream_completes_through_trailing_empty_choices_usage_chunk": ["other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream"], "tests/integration/streaming/test_stream_contracts.py::test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost": ["other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost"], "tests/integration/streaming/test_stream_contracts.py::test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bills_the_fallback": ["other.streaming.fallback.empty_leading_chunk_then_disconnect_streams_fallback_with_usage_and_spend"], "tests/integration/streaming/test_stream_contracts.py::test_responses_stream_completes_through_empty_choices_metadata_and_usage_chunks": ["other.streaming.responses_bridge.empty_choices_chunks_complete_stream"], "tests/integration/streaming/test_stream_parallel_slot_release.py::test_failing_stream_logging_callback_does_not_leak_max_parallel_requests_slot": ["streaming.max_parallel_requests.slot_released_when_stream_logging_callback_fails"], "tests/integration/streaming/test_ttft_keepalive.py::test_stream_emits_sse_ping_comments_before_the_first_data_frame_while_upstream_is_silent": ["streaming.keepalive.sse_pings_fill_silent_time_to_first_token"], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": ["other.observability.callbacks.raising_success_deployment_hook_keeps_response"], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[embeddings]": ["other.observability.callbacks.raising_success_deployment_hook_keeps_response"], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[responses]": ["other.observability.callbacks.raising_success_deployment_hook_keeps_response"], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[videos]": ["other.observability.callbacks.raising_success_deployment_hook_keeps_response"], "tests/integration/management/test_user_updates_wedged_coordination_redis.py::test_user_budget_updates_return_promptly_while_coordination_redis_is_wedged": ["mgmt.user.update.budget_change_returns_promptly_with_wedged_coordination_redis", "mgmt.user.bulk_update.budget_change_returns_promptly_with_wedged_coordination_redis", "mgmt.customer.update.budget_change_returns_promptly_with_wedged_coordination_redis", "mgmt.key.reset_spend.returns_promptly_with_wedged_coordination_redis", "mgmt.auth_cache_invalidation.publish_parked_by_short_redis_wedge_lands_after_recovery", "mgmt.auth_cache_invalidation.burst_with_worker_kill_keeps_serving_while_redis_wedged"]}, "browser": {"tests/e2e/ui/tests/integrationCritical/projectDetachment.spec.ts::project creation and explicit detachment preserve saved scope and restore serving": ["mgmt.key.ui.project_create_clear_preserves_serving_scope"]}} \ No newline at end of file diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 1a7ffd0e8d3..a3dd0590776 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -1,4 +1,5 @@ import base64 +import binascii import json import threading import time @@ -271,9 +272,9 @@ def _span_response_id(span: Mapping[str, object]) -> str | None: if value.startswith("resp_"): try: decoded: Final = base64.b64decode(value[5:] + "===").decode() - except Exception: + except (binascii.Error, UnicodeDecodeError): return value - marker, _, suffix = decoded.partition("response_id:") + _, _, suffix = decoded.partition("response_id:") if suffix: return suffix.split(";", 1)[0] return value @@ -586,9 +587,10 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gatewa for line in stream.iter_lines() if line.startswith("data: ") and line != "data: [DONE]" ) - completed: Final = tuple(event for event in events if event.get("type") == "response.completed") - assert completed, events - attrs: Final = _marked_span(rig.collector, marker, "langfuse.observation.metadata") + response_id: Final = next( + event["response_id"] for event in events if event.get("type") == "response.output_text.delta" + ) + attrs: Final = _observation_span(rig.collector, response_id) _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) @@ -735,8 +737,8 @@ def test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_va _assert_identity(attrs, _chat_identity_fields(identity)) -@pytest.mark.covers("other.observability.langfuse_otel.unauthenticated_request_emits_no_span") -def test_langfuse_otel_unauthenticated_request_emits_no_span(gateway: Gateway, tmp_path: Path) -> None: +@pytest.mark.covers("other.observability.langfuse_otel.unauthenticated_request_leaks_no_identity") +def test_langfuse_otel_unauthenticated_request_leaks_no_identity(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-unauth-" + uuid.uuid4().hex with ( _langfuse_rig(gateway, tmp_path, {}, marker) as rig, @@ -781,16 +783,20 @@ def test_langfuse_otel_unauthenticated_request_emits_no_span(gateway: Gateway, t settled: Final = eventually( watch, lambda spans: sum(_span_response_id(s) == second_id for s in spans) == 1, seconds=25 ) - for span in (span for span in settled if f"{marker}-denied" in json.dumps(span, default=str)): - assert not [name for name in span if name.startswith("langfuse.trace.metadata.user_api_key")], span - leaked: Final = json.loads(str(span.get("langfuse.observation.metadata", "{}"))) - for field in ( - "user_api_key_alias", - "user_api_key_team_id", - "user_api_key_team_alias", - "user_api_key_end_user_id", - ): - assert field not in leaked, leaked + denied_spans: Final = tuple(span for span in settled if f"{marker}-denied" in json.dumps(span, default=str)) + assert len(denied_spans) == 1, [s.get("llm.response.id") for s in settled] + denied_span: Final = denied_spans[0] + assert not [name for name in denied_span if name.startswith("langfuse.trace.metadata.user_api_key")], ( + denied_span + ) + leaked: Final = json.loads(str(denied_span.get("langfuse.observation.metadata", "{}"))) + for field in ( + "user_api_key_alias", + "user_api_key_team_id", + "user_api_key_team_alias", + "user_api_key_end_user_id", + ): + assert field not in leaked, leaked @pytest.mark.covers("other.observability.langfuse_otel.repeated_requests_each_emit_one_span") @@ -831,9 +837,11 @@ def test_langfuse_otel_repeated_identical_requests_emit_one_span_each(gateway: G def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-chaos-outage-" + uuid.uuid4().hex outage: Final = threading.Event() + rejected: Final = [] # mutable-ok: the wire server thread records each 503 it serves def sink_reply(request: Request) -> Reply: if outage.is_set(): + rejected.append(1) return Reply(status=503, body=b'{"error": "sink outage"}') return Reply() @@ -889,8 +897,11 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa while sum(f.done() for f in futures) < 10: wait(futures, timeout=0.05) outage.set() - while sum(f.done() for f in futures) < 25: - wait(futures, timeout=0.05) + eventually( + lambda: (sum(f.done() for f in futures), len(rejected)), + lambda progress: progress[0] >= 25 and progress[1] >= 1, + seconds=20, + ) outage.clear() response_ids: Final = tuple(f.result() for f in futures) assert len(set(response_ids)) == 30, response_ids @@ -902,6 +913,7 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa assert sorted(_span_response_id(span) for span in settled if _span_response_id(span) in response_ids) == sorted( response_ids ) + assert len(rejected) >= 1, "sink outage never served a 503" @pytest.mark.covers("other.observability.langfuse_otel.worker_kill_burst_keeps_serving") @@ -931,13 +943,16 @@ def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: return response.status_code, response.json()["id"] if response.status_code == 200 else response.text workers: Final = tuple(psutil.Process(rig.proxy.process.pid).children(recursive=True)) - assert workers, "owned proxy reported no worker children" + leaves: Final = tuple(worker for worker in workers if not worker.children()) + assert len(leaves) >= 2, f"expected multiple worker leaves, got {workers!r}" + victim: Final = leaves[0] with ThreadPoolExecutor(max_workers=20) as pool: futures: Final = tuple(pool.submit(call, index) for index in range(20)) while sum(f.done() for f in futures) < 5: wait(futures, timeout=0.05) - workers[0].kill() + victim.kill() outcomes: Final = tuple(f.result() for f in futures) + assert not victim.is_running() or victim.status() == psutil.STATUS_ZOMBIE failures: Final = tuple(status for status, _ in outcomes if status != 200) succeeded: Final = tuple(identifier for status, identifier in outcomes if status == 200) assert len(set(succeeded)) == len(succeeded), outcomes From 31b38378071c640dc3107fbb59832bff5df1bd11 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 10:32:52 +0000 Subject: [PATCH 12/28] test(langfuse_otel): restore contracts.json formatting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/contracts.json | 2095 +++++++++++++++++++++++++++++- 1 file changed, 2094 insertions(+), 1 deletion(-) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index c9ec0a6c1e9..358d7ee7cc2 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1 +1,2094 @@ -{"groups": {"management": ["management", "authorization", "configuration"], "accounting": ["pricing", "spend"], "database": ["database"], "providers": ["providers", "routing", "streaming"], "extensions": ["mcp", "observability", "compatibility"], "sdk": ["sdk"], "cost": ["cost_calculation"]}, "tests": {"tests/integration/management/test_key_updates.py::test_update_preserves_independent_fields_and_serving": ["mgmt.key.update.preserves_independent_fields"], "tests/integration/pricing/test_configured_prices.py::test_custom_price_is_reported_and_charged": ["quota_management.spend_tracking.custom_price.matches_input_rates"], "tests/integration/providers/test_request_boundary.py::test_internal_request_state_does_not_reach_provider": ["other.provider_wire.internal_parameters_filtered"], "tests/integration/pricing/test_configured_prices.py::test_default_prices_survive_nullable_sibling_and_reload": ["quota_management.spend_tracking.default_prices.survive_nullable_sibling_reload"], "tests/integration/providers/test_request_boundary.py::test_upstream_rejects_corruption_and_accepts_supported_metadata": ["other.provider_wire.validator_rejects_corruption"], "tests/integration/pricing/test_configured_prices.py::test_loaded_router_preserves_cached_defaults_during_real_requests": ["quota_management.spend_tracking.default_prices.loaded_router_preserves_cached_defaults"], "tests/integration/management/test_partial_update_sequences.py::test_generated_partial_updates_preserve_persisted_and_effective_state": ["mgmt.key.update.generated_sequences_preserve_state"], "tests/integration/management/test_partial_update_sequences.py::test_zero_false_and_empty_values_are_not_treated_as_omission": ["mgmt.key.update.false_zero_and_empty_values_affect_serving"], "tests/integration/management/test_partial_update_sequences.py::test_project_omission_clear_and_invalid_update_have_distinct_effects": ["mgmt.key.update.project_clear_preserves_scope", "mgmt.key.update.invalid_batch_is_atomic"], "tests/integration/authorization/test_warmed_policy.py::test_generated_policy_changes_reach_both_warmed_workers": ["mgmt.key.update.two_workers_enforce_warmed_policy"], "tests/integration/authorization/test_warmed_policy.py::test_scim_deactivation_blocks_null_and_false_keys_but_preserves_other_owners": ["mgmt.user.scim.deactivation_includes_nullable_blocked_keys"], "tests/integration/authorization/test_warmed_policy.py::test_warmed_team_role_demotion_prevents_later_management_writes": ["mgmt.team.member_update.demoted_role_cannot_write"], "tests/integration/configuration/test_effective_settings.py::test_model_block_changes_actual_route_and_leaves_other_route_working": ["mgmt.model.block.changes_serving_and_preserves_control"], "tests/integration/configuration/test_effective_settings.py::test_saved_retry_setting_controls_real_attempts_and_restores": ["mgmt.router_settings.update.changes_observed_attempt_count"], "tests/integration/configuration/test_effective_settings.py::test_credential_value_update_and_model_reload_reach_provider": ["mgmt.credential.update.saved_value_reaches_wire"], "tests/integration/management/test_partial_update_sequences.py::test_denied_key_update_preserves_saved_grants_and_serving": ["mgmt.key.update.denied_request_preserves_effective_state"], "tests/integration/authorization/test_warmed_policy.py::test_expiry_and_explicit_clear_reach_both_warmed_workers": ["mgmt.key.update.expiry_changes_reach_warmed_workers"], "tests/integration/database/test_partition_transactions.py::test_real_partition_ddl_survives_witnessed_lock_and_is_idempotent": ["other.database.partitions.lock_wait_outlives_transaction_default", "other.database.partitions.repeat_preserves_rows"], "tests/integration/database/test_reader_writer_regeneration.py::test_key_regeneration_uses_writer_with_a_real_readonly_reader": ["other.database.regeneration.writer_updates_dependent_grants"], "tests/integration/pricing/test_price_precedence.py::test_generated_zero_null_and_omitted_prices_follow_independent_arithmetic": ["quota_management.spend_tracking.price_precedence.zero_and_default_rates"], "tests/integration/pricing/test_price_precedence.py::test_same_upstream_aliases_keep_distinct_prices_after_reload": ["quota_management.spend_tracking.alias_prices.remain_independent_on_reload"], "tests/integration/pricing/test_off_peak_pricing.py::test_open_off_peak_window_bills_off_peak_rates": ["quota_management.spend_tracking.off_peak_pricing.open_window_bills_off_peak_rates"], "tests/integration/pricing/test_off_peak_pricing.py::test_closed_off_peak_window_bills_standard_rates": ["quota_management.spend_tracking.off_peak_pricing.closed_window_bills_standard_rates"], "tests/integration/spend/test_cache_and_quota.py::test_generated_cache_sequences_preserve_content_usage_and_zero_hit_cost": ["quota_management.response_cache.generated_sequences_preserve_content_and_accounting"], "tests/integration/spend/test_cache_and_quota.py::test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores": ["quota_management.budget.key.boundary_blocks_before_provider_and_reset_restores"], "tests/integration/spend/test_cache_and_quota.py::test_different_system_messages_do_not_share_a_cached_response": ["quota_management.response_cache.system_messages_partition_cache_identity"], "tests/integration/database/test_transaction_atomicity.py::test_access_group_second_key_constraint_failure_rolls_back_all_writes": ["other.database.access_group.failed_second_write_rolls_back_first"], "tests/integration/spend/test_cache_and_quota.py::test_repeated_hits_keep_response_identity_and_create_distinct_zero_cost_rows": ["quota_management.response_cache.repeated_hits_preserve_identity_and_single_charge"], "tests/integration/providers/test_s3_wire.py::test_sigv4_verifier_matches_published_put_and_rejects_corruption": ["other.provider_wire.s3.verifier_known_answer_and_negative_controls"], "tests/integration/providers/test_s3_wire.py::test_s3_sync_and_async_uploads_pass_independent_wire_verification": ["other.provider_wire.s3.sync_async_reserved_keys_are_signed_and_accepted"], "tests/integration/providers/test_bedrock_auth_wire.py::test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials": ["other.provider_wire.bedrock.bearer_sdk_skips_credential_chain"], "tests/integration/providers/test_bedrock_auth_wire.py::test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload": ["other.provider_wire.bedrock.bearer_db_yaml_survives_reload"], "tests/integration/streaming/test_stream_contracts.py::test_generated_tcp_partitions_preserve_unicode_text_identity_and_final_usage": ["other.streaming.byte_partitions.preserve_text_identity_and_usage"], "tests/integration/streaming/test_stream_contracts.py::test_fragmented_tool_names_and_arguments_keep_each_call_identity": ["other.streaming.tools.fragmented_calls_keep_independent_arguments"], "tests/integration/streaming/test_stream_contracts.py::test_proxy_stream_usage_visibility_keeps_exact_persisted_charge": ["other.streaming.usage.client_visibility_preserves_persisted_accounting"], "tests/integration/streaming/test_stream_contracts.py::test_truncated_http_stream_is_an_error_and_next_stream_succeeds": ["other.streaming.failure.truncated_transport_raises_and_control_recovers"], "tests/integration/streaming/test_stream_contracts.py::test_client_cancellation_releases_the_actual_provider_connection": ["other.streaming.cancellation.closes_actual_provider_connection"], "tests/integration/routing/test_observed_routing.py::test_retry_counts_and_public_errors_match_actual_provider_attempts": ["other.routing.retries.several_attempts_reach_success_without_hidden_retries", "other.routing.errors.nonretryable_and_exhausted_failures_remain_errors"], "tests/integration/routing/test_observed_routing.py::test_loaded_fallback_selects_expected_deployment_and_keeps_response_identity": ["other.routing.fallback.loaded_configuration_selects_only_permitted_target"], "tests/integration/routing/test_observed_routing.py::test_saved_deployment_target_update_changes_wire_and_preserves_control": ["other.routing.alias_update.persisted_target_changes_only_selected_route"], "tests/integration/providers/test_bedrock_role_configuration.py::test_role_reference_from_db_and_yaml_reaches_real_sts_http_and_bedrock": ["other.provider_wire.bedrock.db_yaml_role_reference_reaches_sts_and_signed_request"], "tests/integration/routing/test_redis_recovery.py::test_owned_redis_outage_recovers_requests_and_real_response_cache": ["other.routing.redis.owned_outage_recovers_serving_and_response_cache"], "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[type_word]": ["other.provider_wire.anthropic.bare_string_content_item_is_client_error"], "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[plain]": ["other.provider_wire.anthropic.bare_string_content_item_is_client_error"], "tests/integration/providers/test_anthropic_wire.py::test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contracts": ["other.provider_wire.anthropic.tool_history_system_cache_and_internal_fields", "quota_management.spend_tracking.cache_tokens.disjoint_classes_use_explicit_rates"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_create_status_and_content_follow_queue_wire_contract": ["other.provider_wire.fal_ai.video_queue_create_status_and_content_download"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_failed_result_reports_failed_status_and_fal_error": ["other.provider_wire.fal_ai.video_failed_result_surfaces_fal_error"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_auto_duration_omits_duration_and_queues": ["other.provider_wire.fal_ai.h3_auto_duration_omits_duration_and_queues"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_oversized_size_uses_top_resolution_tier_and_queues": ["other.provider_wire.fal_ai.h3_oversized_size_uses_top_resolution_tier"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": ["other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_prices_non_canonical_size_from_nearest_row": ["other.provider_wire.fal_ai.gpt_image_generation_noncanonical_size_uses_nearest_keyed_row"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_sdk_response_honors_dump_options": ["other.provider_wire.fal_ai.sdk_image_response_dump_options"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": ["other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing"], "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_charges_and_polls_pass_through_free": ["other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not"], "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_prices_string_resolution_like_the_integer": ["other.provider_wire.fal_ai.passthrough_queue_submit_prices_string_resolution_like_integer"], "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_to_catalog_key_the_pricer_cannot_price_is_rejected_not_forwarded": ["other.provider_wire.fal_ai.passthrough_queue_submit_rejects_unpriceable_catalog_key"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": ["other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_lora_depth_edit_sends_single_image_url_and_charges_flat_row": ["other.provider_wire.fal_ai.flux_lora_depth_edit_single_image_url_and_flat_pricing"], "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_sends_prompt_image_and_reasoning": ["other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing"], "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_rejects_non_string_reasoning_effort_before_the_wire": ["other.provider_wire.fal_ai.chat_non_string_reasoning_effort_rejected_before_wire"], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_without_deployment_api_base_uses_global_api_base": ["other.provider_wire.fal_ai.global_api_base_routes_image_generation"], "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-pro]": ["other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing"], "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-flash]": ["other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing"], "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas": ["other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas"], "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_tool_call_is_forwarded_and_returned": ["other.provider_wire.xiaomi_mimo.tool_call_survives_translation"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_video_create_uses_canonical_body_and_status_path": ["other.provider_wire.fal_ai.video_queue_create_status_and_content_download"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_carries_the_deployment_extra_headers": ["other.provider_wire.fal_ai.video_result_probe_forwards_extra_headers"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_reuses_the_ssl_verify_false_client": ["other.provider_wire.fal_ai.video_result_probe_honors_ssl_verify"], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_provider_hanging_up_on_the_result_probe_keeps_the_completed_status": ["other.provider_wire.fal_ai.video_result_probe_hangup_stays_completed"], "tests/integration/mcp/test_mcp_lifecycle.py::test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit": ["mcp.call_tool.saved_headers.reach_actual_transport"], "tests/integration/mcp/test_mcp_lifecycle.py::test_tool_error_remains_error_and_healthy_sibling_returns_value": ["mcp.call_tool.errors.tool_failure_is_not_success"], "tests/integration/mcp/test_mcp_lifecycle.py::test_generated_mcp_edits_preserve_actual_headers_and_tool_results": ["other.mcp.lifecycle.generated_save_reload_preserves_effective_headers"], "tests/integration/observability/test_callback_delivery.py::test_concurrent_success_and_failure_join_callbacks_and_rows_without_credentials": ["other.observability.callbacks.credentials_stay_out_of_event_bodies", "other.observability.callbacks.concurrent_results_join_complete_events_and_rows"], "tests/integration/observability/test_otel_text_completion_choices.py::test_otel_weave_output_keeps_text_completion_provider_fields_beside_the_synthesized_message": ["other.observability.otel.text_completion_choices_keep_provider_fields"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys": ["other.observability.langfuse_otel.request_metadata_on_langfuse_keys"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse": ["other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream": ["other.observability.langfuse_otel.request_metadata_openai_sdk_stream"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async": ["other.observability.langfuse_otel.request_metadata_openai_sdk_async"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages": ["other.observability.langfuse_otel.request_metadata_anthropic_sdk_messages"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream": ["other.observability.langfuse_otel.request_metadata_anthropic_sdk_stream"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_api": ["other.observability.langfuse_otel.request_metadata_responses_api"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream": ["other.observability.langfuse_otel.request_metadata_responses_stream_sse"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_caller_trace_metadata_survives_flattened_identity": ["other.observability.langfuse_otel.caller_trace_metadata_preserved_with_flattened_identity"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_failure_span_carries_request_metadata": ["other.observability.langfuse_otel.failure_span_carries_request_metadata"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_emits_no_team_identity": ["other.observability.langfuse_otel.teamless_key_omits_team_identity"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_keeps_caller_trace_metadata": ["other.observability.langfuse_otel.teamless_key_keeps_caller_trace_metadata"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_values": ["other.observability.langfuse_otel.spend_logs_metadata_roundtrips_nonstring_values"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_unauthenticated_request_leaks_no_identity": ["other.observability.langfuse_otel.unauthenticated_request_leaks_no_identity"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_repeated_identical_requests_emit_one_span_each": ["other.observability.langfuse_otel.repeated_requests_each_emit_one_span"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once": ["other.observability.langfuse_otel.sink_outage_burst_lands_each_id_once"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports": ["other.observability.langfuse_otel.worker_kill_burst_keeps_serving"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_slow_sink_does_not_deadlock_exports": ["other.observability.langfuse_otel.slow_sink_does_not_deadlock_exports"], "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_v2_mapper_keeps_existing_trace_metadata_keys": ["other.observability.langfuse_otel.v2_mapper_unchanged"], "tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": ["other.observability.guardrails.rewrite_reaches_correct_anthropic_positions"], "tests/integration/compatibility/test_a2a_wire_versions.py::test_a2a_versions_and_legacy_casing_preserve_real_wire_and_response": ["other.compatibility.a2a.supported_versions_preserve_literal_envelopes"], "tests/integration/compatibility/test_persisted_toolsets.py::test_existing_toolset_format_loads_before_start_and_keeps_sibling_denied": ["other.compatibility.mcp.persisted_tool_names_survive_candidate_startup"], "tests/integration/mcp/test_oauth_configuration.py::test_partial_discovery_and_unrelated_edit_keep_actual_authorization_destination": ["other.mcp.oauth.discovery_cannot_erase_configured_authorization_endpoint"], "tests/integration/observability/test_guardrail_effects.py::test_guardrail_denial_prevents_provider_and_preserves_allowed_control": ["other.observability.guardrails.denial_prevents_provider_with_allowed_control"], "tests/integration/mcp/test_mcp_protocol_errors.py::test_jsonrpc_error_and_malformed_tool_result_remain_errors": ["other.mcp.errors.protocol_and_malformed_results_cannot_be_empty_success"], "tests/integration/compatibility/test_openai_consumer.py::test_retained_openai_clients_parse_real_proxy_tool_and_usage_responses": ["other.compatibility.openai.retained_client_parses_tools_and_usage"], "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": ["quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals"], "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_written_by_the_final_flush": ["quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch"], "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_row_lock_is_written_exactly_once": ["quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch"], "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": ["quota_management.spend_tracking.spend_calculate.rejects_unpriced_model"], "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini-live-2.5-flash-preview-native-audio-09-2025]": ["quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"], "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini/gemini-live-2.5-flash-preview-native-audio-09-2025]": ["quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"], "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": ["mgmt.key.update.project_detach_denied_to_restricted_actor"], "tests/integration/management/test_partial_update_sequences.py::test_cross_tenant_actor_cannot_read_update_or_detach_project_key": ["mgmt.key.info.cross_tenant_key_is_denied", "mgmt.key.update.cross_tenant_key_is_denied", "mgmt.key.update.cross_tenant_project_detach_is_denied"], "tests/integration/management/test_project_lifecycle.py::test_project_new_persists_real_state": ["mgmt.project.new.real_route_persists"], "tests/integration/management/test_project_lifecycle.py::test_project_update_persists_real_state": ["mgmt.project.update.real_route_persists"], "tests/integration/management/test_project_lifecycle.py::test_project_delete_with_attached_key_refuses_and_preserves_state": ["mgmt.project.delete.attached_key_refusal_preserves_state"], "tests/integration/sdk/test_http2_wire.py::test_async_handler_negotiates_http2_only_when_enabled": ["other.sdk_wire.http2.async_handler_negotiates_h2_only_when_enabled"], "tests/integration/sdk/test_http2_wire.py::test_sync_handler_negotiates_http2_only_when_enabled": ["other.sdk_wire.http2.sync_handler_negotiates_h2_only_when_enabled"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-halved_rates_when_map_has_no_batch_keys]": ["quota_management.spend_tracking.batch_costs.fallback_rates"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-cached_input_halved]": ["quota_management.spend_tracking.batch_costs.cached_input"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.4-batch-explicit_batch_rates_bill_cached_at_batch_input_rate]": ["quota_management.spend_tracking.batch_costs.explicit_rates"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-all_requests_failed_zero_spend]": ["quota_management.spend_tracking.batch_costs.failed_requests"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-single_turn_text_audio_cached]": ["quota_management.spend_tracking.realtime_costs.single_turn"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-two_turns_summed_into_one_row]": ["quota_management.spend_tracking.realtime_costs.multiple_turns"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-priced_from_session_created_model]": ["quota_management.spend_tracking.realtime_costs.session_model"], "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-session_without_turns_zero_spend]": ["quota_management.spend_tracking.realtime_costs.session_without_turns"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_native_json]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_429_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-anthropic_us_inference]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_write_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_fast_mode]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_us_inference]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_write_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-anthropic_us_inference]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-fallback_cache_read_at_half_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-image_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-google_maps_grounding]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-fallback_video_tokens_at_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-video_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-web_search_per_prompt]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-google_maps_grounding]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_reasoning_at_output_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_image_tokens_at_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-image_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-video_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-google_maps_grounding]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-image_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-video_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-web_search_per_prompt]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-google_maps_grounding]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_prompt_blocked]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-file_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_incomplete]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_incomplete]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_unvalidated]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_unvalidated]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-file_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_incomplete]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_incomplete]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_unvalidated]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_unvalidated]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_output]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_high]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_read_at_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_write_at_input_rate]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_read_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_write_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_image_input]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_response_model_override]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_tool_call]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-next-transcriptions-per-second]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-verbose-next-transcriptions-duration]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-4o-transcribe-next-transcriptions-tokens]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[nova-next-transcriptions-per-second]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-whisper-next-transcriptions-deployment]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-speech-per-character]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-hd-speech-per-character]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-tts-next-speech-deployment]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-standard]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-hd]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-wide]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-two]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-low]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[imagen-next-images-one]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[amazon-nova-canvas-next-images-one]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-edit]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embeddings-4-large-deployment]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-embeddings-v4]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-rerank-v4]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-embeddings-titan-v2]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-embeddings-v5]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-one]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-three]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-total-tokens-fallback]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks-embeddings-v1]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-embeddings-002]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-list]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-single]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-basic]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-n-best]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-stream-usage]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-3-large-dimensions]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-batch]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-single]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-token-array]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-completions-v1]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-embeddings-v1]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[vertex-embeddings-text-006]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_reasoning]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream_cache_read]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_incomplete]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_previous_response_id]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_web_search_medium]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-responses_file_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_flex]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_priority]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_5m]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_1h]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_web_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream_cache_read]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_tiered_input_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-messages_input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-messages_input_text]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-messages_cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-passthrough-generate_content_priced_via_gemini_key]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-passthrough-stream_generate_content_priced_via_vertex_key]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-passthrough-messages]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-passthrough-messages_cache_read]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse_stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_input]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_boundary_stays_lower_tier]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_second_tier]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_above_top_range]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_below_128k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_above_128k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_creation_1h_above_200k]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-provider_reported_cost]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-token_priced]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-no_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-prompt_cache_hit]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-no_cache_fields_bills_zero_cache]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-reasoning_folded_into_completion]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-live_search]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-provider_reported_cost]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-invoke-haiku-json]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-invoke-haiku-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-profile-base-model]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-eu-regional-key]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-apac-bare-fallback]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-nova-2-pro]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-mistral-large-3-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-ai-gpt-5.4-mini-latest]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-ai-gpt-5.4-mini-latest-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-pinned-gpt-5.4-mini-stream]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[groq-qwen-3.8-json]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[groq-qwen-3.8-stream_x_groq_recount]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-command-a-v2-tokens]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[mistral-medium-2604-json]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openai-deployment-pricing-override]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_400_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_401_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_stream_request_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_upstream_500_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_upstream_500_zero_spend]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-fallback_billed_to_answering_deployment]": ["quota_management.spend_tracking.routing.fallback_billing"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-n_2_choices]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-finish_reason_length]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_usage_in_empty_choices_chunk]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_usage_in_last_delta_chunk]": ["quota_management.spend_tracking.scripted_wire.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-unknown_model_response_model_unknown]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-unknown_model_response_model_known]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-chat_request_to_embedding_entry]": ["quota_management.spend_tracking.cost_matrix.logs_cost"], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-client_disconnect_mid_stream]": ["quota_management.spend_tracking.scripted_wire.client_disconnect"], "tests/integration/mcp/test_mcp_lifecycle.py::test_health_intersects_route_restricted_key_grants_in_both_management_modes": ["other.mcp.health.restricted_keys_intersect_grants_in_both_modes"], "tests/integration/mcp/test_mcp_lifecycle.py::test_warm_credential_removal_rejects_without_upstream_traffic": ["other.mcp.credentials.warm_removal_fails_closed_without_upstream_traffic"], "tests/integration/observability/test_guardrail_effects.py::test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls": ["other.mcp.guardrails.request_selection_blocks_resolved_tool_without_execution"], "tests/integration/mcp/test_oauth_configuration.py::test_same_url_oauth_credentials_and_revocation_are_isolated_by_user_and_server[revoke]": ["other.mcp.oauth.same_url_credentials_are_isolated_by_user_and_server"], "tests/integration/mcp/test_oauth_configuration.py::test_same_url_oauth_credentials_and_revocation_are_isolated_by_user_and_server[expire]": ["other.mcp.oauth.same_url_credentials_are_isolated_by_user_and_server"], "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[anonymous]": ["other.mcp.permissions.same_url_servers_enforce_discovery_and_execution"], "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[bearer]": ["other.mcp.permissions.same_url_servers_enforce_discovery_and_execution"], "tests/integration/management/test_budget_updates.py::test_shortening_budget_duration_moves_reset_at_onto_the_new_schedule": ["mgmt.budget.update.duration_change_recomputes_reset_at"], "tests/integration/management/test_organization_budget_clear.py::test_patch_organization_update_with_null_tpm_limit_clears_it_and_keeps_sibling_limits": ["mgmt.organization.update.null_clears_budget_limit"], "tests/integration/management/test_team_budget_duration_defaults.py::test_team_new_explicit_null_budget_duration_is_not_replaced_by_default": ["mgmt.team.new.explicit_null_budget_duration_overrides_default"], "tests/integration/management/test_team_member_budget_cache.py::test_team_member_default_budget_lands_in_redis_after_first_member_call": ["mgmt.team_member_budget.default_budget_is_cached_in_redis_as_json"], "tests/integration/observability/test_callback_delivery.py::test_streamed_responses_success_callback_carries_provider_apim_request_id": ["other.observability.callbacks.streamed_responses_events_carry_provider_response_headers"], "tests/integration/observability/test_guardrail_effects.py::test_bedrock_passthrough_converse_guardrail_ignores_denied_term_in_tool_definition": ["other.observability.guardrails.bedrock_passthrough_converse_scans_only_caller_content"], "tests/integration/pricing/test_configured_prices.py::test_cost_estimate_reports_configured_prices_for_model_absent_from_cost_map": ["quota_management.cost_estimate.configured_price.reported_for_model_absent_from_cost_map"], "tests/integration/pricing/test_configured_prices.py::test_saving_echoed_model_info_does_not_freeze_cost_map_price_into_deployment": ["pricing.model_update.echoed_cost_map_price_is_not_persisted_as_override"], "tests/integration/pricing/test_databricks_cache_pricing.py::test_databricks_cached_prompt_tokens_bill_at_cache_rates_not_input_rate": ["pricing.databricks.cached_prompt_tokens_bill_at_cache_rates"], "tests/integration/pricing/test_ocr_page_pricing.py::test_ocr_annotation_pages_are_billed_at_annotation_cost_per_page": ["pricing.ocr.annotation_pages_billed_at_annotation_rate"], "tests/integration/pricing/test_service_tier_pricing.py::test_ultrafast_service_tier_bills_ultrafast_rates_and_keeps_pricing_off_the_wire": ["quota_management.spend_tracking.service_tier_pricing.ultrafast_bills_ultrafast_rates"], "tests/integration/spend/test_batch_completion_accounting.py::test_completed_batch_spend_row_records_reasoning_tokens_and_error_file_failures": ["quota_management.spend_tracking.batch_costs.reasoning_tokens_and_error_file_failures_recorded"], "tests/integration/spend/test_batch_observability.py::test_batch_retrieval_row_sums_reasoning_tokens_and_counts_output_and_error_file_failures": ["spend.batches.retrieval_row_aggregates_reasoning_tokens_and_per_request_counts"], "tests/integration/spend/test_batch_poll_starvation.py::test_batches_gone_at_provider_do_not_starve_a_newer_batch_out_of_cost_polling": ["quota_management.spend_tracking.batch_costs.uncostable_rows_retire_so_newer_batches_are_costed"], "tests/integration/spend/test_cache_and_quota.py::test_in_flight_count_tokens_does_not_reserve_key_budget_away_from_a_completion": ["quota_management.budget.key.in_flight_count_tokens_reserves_nothing_so_completion_reaches_provider"], "tests/integration/spend/test_cache_and_quota.py::test_repeated_count_tokens_on_budgeted_key_does_not_reserve_budget_or_block_later_completion": ["quota_management.budget.key.count_tokens_reserves_nothing_so_completion_within_budget_succeeds"], "tests/integration/spend/test_daily_rollup_retry.py::test_failed_daily_user_rollup_commit_is_retried_so_spend_report_and_daily_activity_agree": ["spend.daily_rollup.failed_user_commit_is_retried_until_report_and_daily_activity_agree"], "tests/integration/spend/test_disconnected_bedrock_messages_stream_billing.py::test_client_disconnect_mid_bedrock_messages_stream_still_bills_terminal_usage": ["spend.anthropic_messages_stream.client_disconnect_bills_terminal_bedrock_usage"], "tests/integration/spend/test_failed_dispatch_tokens.py::test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_row": ["spend.failed_dispatch.failure_row_records_estimated_input_tokens"], "tests/integration/spend/test_model_router_selected_model.py::test_model_router_alias_without_router_in_name_keeps_selected_model_in_response_and_spend_log": ["spend.model_router.selected_model_is_returned_and_persisted_for_plain_alias"], "tests/integration/spend/test_org_budget_cli_session_token.py::test_cli_session_token_without_org_id_charges_and_caps_the_team_organization": ["quota_management.organization_budget.cli_session_token_without_org_id_charges_team_organization"], "tests/integration/spend/test_passthrough_budget_reservation.py::test_repeated_gemini_passthrough_calls_stay_served_while_key_spend_is_below_max_budget": ["spend.budget_reservation.gemini_passthrough_success_releases_reservation_from_spend_counter"], "tests/integration/spend/test_team_daily_activity_aggregated.py::test_aggregated_team_activity_reports_the_whole_range_team_spend_in_one_page": ["quota_management.spend_tracking.team_daily_activity_aggregated_reports_whole_range_team_spend"], "tests/integration/spend/test_team_member_spend.py::test_member_added_without_any_budget_is_charged_on_its_membership_row": ["spend.team_member.member_without_budget_gets_membership_row_and_spend"], "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_is_listed_beside_db_store_and_survives_listing": ["mgmt.vector_store.list.keeps_config_store_beside_db_stores"], "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_refuses_new_update_and_delete": ["mgmt.vector_store.write.config_store_is_read_only"], "tests/integration/management/test_vector_store_config_ownership.py::test_db_store_lifecycle_is_unchanged_beside_config_store": ["mgmt.vector_store.write.db_store_lifecycle_unchanged_beside_config_store"], "tests/integration/management/test_vector_store_config_ownership.py::test_chat_with_config_store_searches_upstream_and_injects_context_after_listing": ["other.vector_store.chat.config_store_search_reaches_upstream_after_listing"], "tests/integration/management/test_vector_store_config_ownership.py::test_passthrough_search_on_config_store_uses_yaml_credentials_after_listing": ["other.vector_store.search.config_store_passthrough_uses_yaml_credentials_after_listing"], "tests/integration/management/test_vector_store_config_ownership.py::test_non_admin_key_access_to_config_store_follows_grants_after_admin_listing": ["authz.vector_store.list.non_admin_key_access_to_config_store_follows_grants"], "tests/integration/management/test_vector_store_config_ownership.py::test_peer_process_keeps_config_store_and_sees_db_store_created_elsewhere": ["mgmt.vector_store.list.peer_process_keeps_config_store_and_sees_db_store"], "tests/integration/management/test_vector_store_config_ownership.py::test_concurrent_burst_keeps_config_store_and_refuses_every_config_write": ["mgmt.vector_store.chaos.concurrent_burst_keeps_config_store_across_workers"], "tests/integration/management/test_vector_store_config_ownership.py::test_redis_outage_keeps_config_store_served_and_recovers": ["mgmt.vector_store.chaos.redis_outage_keeps_config_store_and_recovers"], "tests/integration/providers/test_anthropic_advisor_wire.py::test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated": ["providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment"], "tests/integration/providers/test_azure_ai_chat_wire.py::test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages": ["providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry"], "tests/integration/providers/test_azure_ai_flux2_image_wire.py::test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro": ["other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body"], "tests/integration/providers/test_azure_ai_rerank_auth_wire.py::test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider": ["other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider"], "tests/integration/providers/test_bedrock_auth_wire.py::test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature": ["providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization"], "tests/integration/providers/test_bedrock_batch_files_wire.py::test_completions_and_responses_batch_records_upload_as_anthropic_user_messages": ["other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages"], "tests/integration/providers/test_bedrock_claude_thinking_wire.py::test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens": ["other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking"], "tests/integration/providers/test_bedrock_converse_config_blocks_wire.py::test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config": ["other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level"], "tests/integration/providers/test_bedrock_embedding_wire.py::test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions": ["other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format"], "tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py::test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort": ["providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort"], "tests/integration/providers/test_bedrock_invoke_tool_search_wire.py::test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field": ["providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support"], "tests/integration/providers/test_bedrock_mantle_codex_input_wire.py::test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items": ["other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types"], "tests/integration/providers/test_bedrock_mantle_responses_wire.py::test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle": ["providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items"], "tests/integration/providers/test_bedrock_mantle_wire.py::test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long": ["other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long"], "tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py::test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers": ["providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text"], "tests/integration/providers/test_bedrock_passthrough_stream_wire.py::test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type": ["other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type"], "tests/integration/providers/test_bedrock_rerank_wire.py::test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature": ["providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned"], "tests/integration/providers/test_bedrock_thinking_tokens_wire.py::test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens": ["other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text"], "tests/integration/providers/test_dashscope_chat_wire.py::test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider": ["other.provider_wire.dashscope.reasoning_effort_reaches_provider"], "tests/integration/providers/test_databricks_chat_wire.py::test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log": ["other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log"], "tests/integration/providers/test_databricks_oauth_wire.py::test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin": ["other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base"], "tests/integration/providers/test_deepseek_vision_wire.py::test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text": ["other.provider_wire.deepseek.vision_image_content_list_reaches_provider"], "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_chat_sends_router_resource_not_models_path": ["other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name"], "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path": ["other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name"], "tests/integration/providers/test_openai_chat_wire.py::test_openai_chat_tool_choice_without_tools_is_not_forwarded": ["providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire"], "tests/integration/providers/test_openai_image_edit_wire.py::test_openai_compatible_image_edit_forwards_seed_form_field_to_backend": ["other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields"], "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out": ["other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length"], "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[reasoning_effort_none]": ["other.provider_wire.tencent.thinking_reaches_provider_in_request_body"], "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[thinking_enabled]": ["other.provider_wire.tencent.thinking_reaches_provider_in_request_body"], "tests/integration/providers/test_websearch_interception_wire.py::test_capped_websearch_interception_loop_ends_turn_instead_of_exposing_internal_tool_use": ["other.provider_wire.anthropic.websearch_interception_capped_loop_ends_turn_without_internal_tool_use"], "tests/integration/providers/test_websearch_interception_wire.py::test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks": ["other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results"], "tests/integration/providers/test_xai_web_search_wire.py::test_xai_chat_web_search_is_sent_to_responses_with_instructions_and_nested_filters": ["other.provider_wire.xai.chat_web_search_reaches_responses_with_instructions_and_filters"], "tests/integration/routing/test_priority_rate_limit_headers.py::test_non_streaming_v1_messages_success_carries_v3_priority_rate_limit_headers": ["other.routing.priority_rate_limits.v1_messages_success_exposes_v3_priority_headers"], "tests/integration/routing/test_stale_cost_map_boot.py::test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_reload": ["other.routing.cost_map.config_deployment_dropped_by_stale_boot_map_is_restored_after_reload"], "tests/integration/streaming/test_stream_contracts.py::test_messages_stream_completes_through_trailing_empty_choices_usage_chunk": ["other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream"], "tests/integration/streaming/test_stream_contracts.py::test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost": ["other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost"], "tests/integration/streaming/test_stream_contracts.py::test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bills_the_fallback": ["other.streaming.fallback.empty_leading_chunk_then_disconnect_streams_fallback_with_usage_and_spend"], "tests/integration/streaming/test_stream_contracts.py::test_responses_stream_completes_through_empty_choices_metadata_and_usage_chunks": ["other.streaming.responses_bridge.empty_choices_chunks_complete_stream"], "tests/integration/streaming/test_stream_parallel_slot_release.py::test_failing_stream_logging_callback_does_not_leak_max_parallel_requests_slot": ["streaming.max_parallel_requests.slot_released_when_stream_logging_callback_fails"], "tests/integration/streaming/test_ttft_keepalive.py::test_stream_emits_sse_ping_comments_before_the_first_data_frame_while_upstream_is_silent": ["streaming.keepalive.sse_pings_fill_silent_time_to_first_token"], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": ["other.observability.callbacks.raising_success_deployment_hook_keeps_response"], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[embeddings]": ["other.observability.callbacks.raising_success_deployment_hook_keeps_response"], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[responses]": ["other.observability.callbacks.raising_success_deployment_hook_keeps_response"], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[videos]": ["other.observability.callbacks.raising_success_deployment_hook_keeps_response"], "tests/integration/management/test_user_updates_wedged_coordination_redis.py::test_user_budget_updates_return_promptly_while_coordination_redis_is_wedged": ["mgmt.user.update.budget_change_returns_promptly_with_wedged_coordination_redis", "mgmt.user.bulk_update.budget_change_returns_promptly_with_wedged_coordination_redis", "mgmt.customer.update.budget_change_returns_promptly_with_wedged_coordination_redis", "mgmt.key.reset_spend.returns_promptly_with_wedged_coordination_redis", "mgmt.auth_cache_invalidation.publish_parked_by_short_redis_wedge_lands_after_recovery", "mgmt.auth_cache_invalidation.burst_with_worker_kill_keeps_serving_while_redis_wedged"]}, "browser": {"tests/e2e/ui/tests/integrationCritical/projectDetachment.spec.ts::project creation and explicit detachment preserve saved scope and restore serving": ["mgmt.key.ui.project_create_clear_preserves_serving_scope"]}} \ No newline at end of file +{ + "groups": { + "management": [ + "management", + "authorization", + "configuration" + ], + "accounting": [ + "pricing", + "spend" + ], + "database": [ + "database" + ], + "providers": [ + "providers", + "routing", + "streaming" + ], + "extensions": [ + "mcp", + "observability", + "compatibility" + ], + "sdk": [ + "sdk" + ], + "cost": [ + "cost_calculation" + ] + }, + "tests": { + "tests/integration/management/test_key_updates.py::test_update_preserves_independent_fields_and_serving": [ + "mgmt.key.update.preserves_independent_fields" + ], + "tests/integration/pricing/test_configured_prices.py::test_custom_price_is_reported_and_charged": [ + "quota_management.spend_tracking.custom_price.matches_input_rates" + ], + "tests/integration/providers/test_request_boundary.py::test_internal_request_state_does_not_reach_provider": [ + "other.provider_wire.internal_parameters_filtered" + ], + "tests/integration/pricing/test_configured_prices.py::test_default_prices_survive_nullable_sibling_and_reload": [ + "quota_management.spend_tracking.default_prices.survive_nullable_sibling_reload" + ], + "tests/integration/providers/test_request_boundary.py::test_upstream_rejects_corruption_and_accepts_supported_metadata": [ + "other.provider_wire.validator_rejects_corruption" + ], + "tests/integration/pricing/test_configured_prices.py::test_loaded_router_preserves_cached_defaults_during_real_requests": [ + "quota_management.spend_tracking.default_prices.loaded_router_preserves_cached_defaults" + ], + "tests/integration/management/test_partial_update_sequences.py::test_generated_partial_updates_preserve_persisted_and_effective_state": [ + "mgmt.key.update.generated_sequences_preserve_state" + ], + "tests/integration/management/test_partial_update_sequences.py::test_zero_false_and_empty_values_are_not_treated_as_omission": [ + "mgmt.key.update.false_zero_and_empty_values_affect_serving" + ], + "tests/integration/management/test_partial_update_sequences.py::test_project_omission_clear_and_invalid_update_have_distinct_effects": [ + "mgmt.key.update.project_clear_preserves_scope", + "mgmt.key.update.invalid_batch_is_atomic" + ], + "tests/integration/authorization/test_warmed_policy.py::test_generated_policy_changes_reach_both_warmed_workers": [ + "mgmt.key.update.two_workers_enforce_warmed_policy" + ], + "tests/integration/authorization/test_warmed_policy.py::test_scim_deactivation_blocks_null_and_false_keys_but_preserves_other_owners": [ + "mgmt.user.scim.deactivation_includes_nullable_blocked_keys" + ], + "tests/integration/authorization/test_warmed_policy.py::test_warmed_team_role_demotion_prevents_later_management_writes": [ + "mgmt.team.member_update.demoted_role_cannot_write" + ], + "tests/integration/configuration/test_effective_settings.py::test_model_block_changes_actual_route_and_leaves_other_route_working": [ + "mgmt.model.block.changes_serving_and_preserves_control" + ], + "tests/integration/configuration/test_effective_settings.py::test_saved_retry_setting_controls_real_attempts_and_restores": [ + "mgmt.router_settings.update.changes_observed_attempt_count" + ], + "tests/integration/configuration/test_effective_settings.py::test_credential_value_update_and_model_reload_reach_provider": [ + "mgmt.credential.update.saved_value_reaches_wire" + ], + "tests/integration/management/test_partial_update_sequences.py::test_denied_key_update_preserves_saved_grants_and_serving": [ + "mgmt.key.update.denied_request_preserves_effective_state" + ], + "tests/integration/authorization/test_warmed_policy.py::test_expiry_and_explicit_clear_reach_both_warmed_workers": [ + "mgmt.key.update.expiry_changes_reach_warmed_workers" + ], + "tests/integration/database/test_partition_transactions.py::test_real_partition_ddl_survives_witnessed_lock_and_is_idempotent": [ + "other.database.partitions.lock_wait_outlives_transaction_default", + "other.database.partitions.repeat_preserves_rows" + ], + "tests/integration/database/test_reader_writer_regeneration.py::test_key_regeneration_uses_writer_with_a_real_readonly_reader": [ + "other.database.regeneration.writer_updates_dependent_grants" + ], + "tests/integration/pricing/test_price_precedence.py::test_generated_zero_null_and_omitted_prices_follow_independent_arithmetic": [ + "quota_management.spend_tracking.price_precedence.zero_and_default_rates" + ], + "tests/integration/pricing/test_price_precedence.py::test_same_upstream_aliases_keep_distinct_prices_after_reload": [ + "quota_management.spend_tracking.alias_prices.remain_independent_on_reload" + ], + "tests/integration/pricing/test_off_peak_pricing.py::test_open_off_peak_window_bills_off_peak_rates": [ + "quota_management.spend_tracking.off_peak_pricing.open_window_bills_off_peak_rates" + ], + "tests/integration/pricing/test_off_peak_pricing.py::test_closed_off_peak_window_bills_standard_rates": [ + "quota_management.spend_tracking.off_peak_pricing.closed_window_bills_standard_rates" + ], + "tests/integration/spend/test_cache_and_quota.py::test_generated_cache_sequences_preserve_content_usage_and_zero_hit_cost": [ + "quota_management.response_cache.generated_sequences_preserve_content_and_accounting" + ], + "tests/integration/spend/test_cache_and_quota.py::test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores": [ + "quota_management.budget.key.boundary_blocks_before_provider_and_reset_restores" + ], + "tests/integration/spend/test_cache_and_quota.py::test_different_system_messages_do_not_share_a_cached_response": [ + "quota_management.response_cache.system_messages_partition_cache_identity" + ], + "tests/integration/database/test_transaction_atomicity.py::test_access_group_second_key_constraint_failure_rolls_back_all_writes": [ + "other.database.access_group.failed_second_write_rolls_back_first" + ], + "tests/integration/spend/test_cache_and_quota.py::test_repeated_hits_keep_response_identity_and_create_distinct_zero_cost_rows": [ + "quota_management.response_cache.repeated_hits_preserve_identity_and_single_charge" + ], + "tests/integration/providers/test_s3_wire.py::test_sigv4_verifier_matches_published_put_and_rejects_corruption": [ + "other.provider_wire.s3.verifier_known_answer_and_negative_controls" + ], + "tests/integration/providers/test_s3_wire.py::test_s3_sync_and_async_uploads_pass_independent_wire_verification": [ + "other.provider_wire.s3.sync_async_reserved_keys_are_signed_and_accepted" + ], + "tests/integration/providers/test_bedrock_auth_wire.py::test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials": [ + "other.provider_wire.bedrock.bearer_sdk_skips_credential_chain" + ], + "tests/integration/providers/test_bedrock_auth_wire.py::test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload": [ + "other.provider_wire.bedrock.bearer_db_yaml_survives_reload" + ], + "tests/integration/streaming/test_stream_contracts.py::test_generated_tcp_partitions_preserve_unicode_text_identity_and_final_usage": [ + "other.streaming.byte_partitions.preserve_text_identity_and_usage" + ], + "tests/integration/streaming/test_stream_contracts.py::test_fragmented_tool_names_and_arguments_keep_each_call_identity": [ + "other.streaming.tools.fragmented_calls_keep_independent_arguments" + ], + "tests/integration/streaming/test_stream_contracts.py::test_proxy_stream_usage_visibility_keeps_exact_persisted_charge": [ + "other.streaming.usage.client_visibility_preserves_persisted_accounting" + ], + "tests/integration/streaming/test_stream_contracts.py::test_truncated_http_stream_is_an_error_and_next_stream_succeeds": [ + "other.streaming.failure.truncated_transport_raises_and_control_recovers" + ], + "tests/integration/streaming/test_stream_contracts.py::test_client_cancellation_releases_the_actual_provider_connection": [ + "other.streaming.cancellation.closes_actual_provider_connection" + ], + "tests/integration/routing/test_observed_routing.py::test_retry_counts_and_public_errors_match_actual_provider_attempts": [ + "other.routing.retries.several_attempts_reach_success_without_hidden_retries", + "other.routing.errors.nonretryable_and_exhausted_failures_remain_errors" + ], + "tests/integration/routing/test_observed_routing.py::test_loaded_fallback_selects_expected_deployment_and_keeps_response_identity": [ + "other.routing.fallback.loaded_configuration_selects_only_permitted_target" + ], + "tests/integration/routing/test_observed_routing.py::test_saved_deployment_target_update_changes_wire_and_preserves_control": [ + "other.routing.alias_update.persisted_target_changes_only_selected_route" + ], + "tests/integration/providers/test_bedrock_role_configuration.py::test_role_reference_from_db_and_yaml_reaches_real_sts_http_and_bedrock": [ + "other.provider_wire.bedrock.db_yaml_role_reference_reaches_sts_and_signed_request" + ], + "tests/integration/routing/test_redis_recovery.py::test_owned_redis_outage_recovers_requests_and_real_response_cache": [ + "other.routing.redis.owned_outage_recovers_serving_and_response_cache" + ], + "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[type_word]": [ + "other.provider_wire.anthropic.bare_string_content_item_is_client_error" + ], + "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[plain]": [ + "other.provider_wire.anthropic.bare_string_content_item_is_client_error" + ], + "tests/integration/providers/test_anthropic_wire.py::test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contracts": [ + "other.provider_wire.anthropic.tool_history_system_cache_and_internal_fields", + "quota_management.spend_tracking.cache_tokens.disjoint_classes_use_explicit_rates" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_create_status_and_content_follow_queue_wire_contract": [ + "other.provider_wire.fal_ai.video_queue_create_status_and_content_download" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_failed_result_reports_failed_status_and_fal_error": [ + "other.provider_wire.fal_ai.video_failed_result_surfaces_fal_error" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_auto_duration_omits_duration_and_queues": [ + "other.provider_wire.fal_ai.h3_auto_duration_omits_duration_and_queues" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_oversized_size_uses_top_resolution_tier_and_queues": [ + "other.provider_wire.fal_ai.h3_oversized_size_uses_top_resolution_tier" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": [ + "other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_prices_non_canonical_size_from_nearest_row": [ + "other.provider_wire.fal_ai.gpt_image_generation_noncanonical_size_uses_nearest_keyed_row" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_sdk_response_honors_dump_options": [ + "other.provider_wire.fal_ai.sdk_image_response_dump_options" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": [ + "other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing" + ], + "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_charges_and_polls_pass_through_free": [ + "other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not" + ], + "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_prices_string_resolution_like_the_integer": [ + "other.provider_wire.fal_ai.passthrough_queue_submit_prices_string_resolution_like_integer" + ], + "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_to_catalog_key_the_pricer_cannot_price_is_rejected_not_forwarded": [ + "other.provider_wire.fal_ai.passthrough_queue_submit_rejects_unpriceable_catalog_key" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": [ + "other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_lora_depth_edit_sends_single_image_url_and_charges_flat_row": [ + "other.provider_wire.fal_ai.flux_lora_depth_edit_single_image_url_and_flat_pricing" + ], + "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_sends_prompt_image_and_reasoning": [ + "other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing" + ], + "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_rejects_non_string_reasoning_effort_before_the_wire": [ + "other.provider_wire.fal_ai.chat_non_string_reasoning_effort_rejected_before_wire" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_without_deployment_api_base_uses_global_api_base": [ + "other.provider_wire.fal_ai.global_api_base_routes_image_generation" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-pro]": [ + "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-flash]": [ + "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas": [ + "other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_tool_call_is_forwarded_and_returned": [ + "other.provider_wire.xiaomi_mimo.tool_call_survives_translation" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_video_create_uses_canonical_body_and_status_path": [ + "other.provider_wire.fal_ai.video_queue_create_status_and_content_download" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_carries_the_deployment_extra_headers": [ + "other.provider_wire.fal_ai.video_result_probe_forwards_extra_headers" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_reuses_the_ssl_verify_false_client": [ + "other.provider_wire.fal_ai.video_result_probe_honors_ssl_verify" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_provider_hanging_up_on_the_result_probe_keeps_the_completed_status": [ + "other.provider_wire.fal_ai.video_result_probe_hangup_stays_completed" + ], + "tests/integration/mcp/test_mcp_lifecycle.py::test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit": [ + "mcp.call_tool.saved_headers.reach_actual_transport" + ], + "tests/integration/mcp/test_mcp_lifecycle.py::test_tool_error_remains_error_and_healthy_sibling_returns_value": [ + "mcp.call_tool.errors.tool_failure_is_not_success" + ], + "tests/integration/mcp/test_mcp_lifecycle.py::test_generated_mcp_edits_preserve_actual_headers_and_tool_results": [ + "other.mcp.lifecycle.generated_save_reload_preserves_effective_headers" + ], + "tests/integration/observability/test_callback_delivery.py::test_concurrent_success_and_failure_join_callbacks_and_rows_without_credentials": [ + "other.observability.callbacks.credentials_stay_out_of_event_bodies", + "other.observability.callbacks.concurrent_results_join_complete_events_and_rows" + ], + "tests/integration/observability/test_otel_text_completion_choices.py::test_otel_weave_output_keeps_text_completion_provider_fields_beside_the_synthesized_message": [ + "other.observability.otel.text_completion_choices_keep_provider_fields" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys": [ + "other.observability.langfuse_otel.request_metadata_on_langfuse_keys" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse": [ + "other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream": [ + "other.observability.langfuse_otel.request_metadata_openai_sdk_stream" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async": [ + "other.observability.langfuse_otel.request_metadata_openai_sdk_async" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages": [ + "other.observability.langfuse_otel.request_metadata_anthropic_sdk_messages" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream": [ + "other.observability.langfuse_otel.request_metadata_anthropic_sdk_stream" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_api": [ + "other.observability.langfuse_otel.request_metadata_responses_api" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream": [ + "other.observability.langfuse_otel.request_metadata_responses_stream_sse" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_caller_trace_metadata_survives_flattened_identity": [ + "other.observability.langfuse_otel.caller_trace_metadata_preserved_with_flattened_identity" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_failure_span_carries_request_metadata": [ + "other.observability.langfuse_otel.failure_span_carries_request_metadata" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_emits_no_team_identity": [ + "other.observability.langfuse_otel.teamless_key_omits_team_identity" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_teamless_key_keeps_caller_trace_metadata": [ + "other.observability.langfuse_otel.teamless_key_keeps_caller_trace_metadata" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_values": [ + "other.observability.langfuse_otel.spend_logs_metadata_roundtrips_nonstring_values" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_unauthenticated_request_leaks_no_identity": [ + "other.observability.langfuse_otel.unauthenticated_request_leaks_no_identity" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_repeated_identical_requests_emit_one_span_each": [ + "other.observability.langfuse_otel.repeated_requests_each_emit_one_span" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once": [ + "other.observability.langfuse_otel.sink_outage_burst_lands_each_id_once" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports": [ + "other.observability.langfuse_otel.worker_kill_burst_keeps_serving" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_slow_sink_does_not_deadlock_exports": [ + "other.observability.langfuse_otel.slow_sink_does_not_deadlock_exports" + ], + "tests/integration/observability/test_langfuse_otel_metadata.py::test_langfuse_otel_v2_mapper_keeps_existing_trace_metadata_keys": [ + "other.observability.langfuse_otel.v2_mapper_unchanged" + ], + "tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": [ + "other.observability.guardrails.rewrite_reaches_correct_anthropic_positions" + ], + "tests/integration/compatibility/test_a2a_wire_versions.py::test_a2a_versions_and_legacy_casing_preserve_real_wire_and_response": [ + "other.compatibility.a2a.supported_versions_preserve_literal_envelopes" + ], + "tests/integration/compatibility/test_persisted_toolsets.py::test_existing_toolset_format_loads_before_start_and_keeps_sibling_denied": [ + "other.compatibility.mcp.persisted_tool_names_survive_candidate_startup" + ], + "tests/integration/mcp/test_oauth_configuration.py::test_partial_discovery_and_unrelated_edit_keep_actual_authorization_destination": [ + "other.mcp.oauth.discovery_cannot_erase_configured_authorization_endpoint" + ], + "tests/integration/observability/test_guardrail_effects.py::test_guardrail_denial_prevents_provider_and_preserves_allowed_control": [ + "other.observability.guardrails.denial_prevents_provider_with_allowed_control" + ], + "tests/integration/mcp/test_mcp_protocol_errors.py::test_jsonrpc_error_and_malformed_tool_result_remain_errors": [ + "other.mcp.errors.protocol_and_malformed_results_cannot_be_empty_success" + ], + "tests/integration/compatibility/test_openai_consumer.py::test_retained_openai_clients_parse_real_proxy_tool_and_usage_responses": [ + "other.compatibility.openai.retained_client_parses_tools_and_usage" + ], + "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [ + "quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals" + ], + "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_written_by_the_final_flush": [ + "quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch" + ], + "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_row_lock_is_written_exactly_once": [ + "quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch" + ], + "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [ + "quota_management.spend_tracking.spend_calculate.rejects_unpriced_model" + ], + "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini-live-2.5-flash-preview-native-audio-09-2025]": [ + "quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate" + ], + "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini/gemini-live-2.5-flash-preview-native-audio-09-2025]": [ + "quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate" + ], + "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [ + "mgmt.key.update.project_detach_denied_to_restricted_actor" + ], + "tests/integration/management/test_partial_update_sequences.py::test_cross_tenant_actor_cannot_read_update_or_detach_project_key": [ + "mgmt.key.info.cross_tenant_key_is_denied", + "mgmt.key.update.cross_tenant_key_is_denied", + "mgmt.key.update.cross_tenant_project_detach_is_denied" + ], + "tests/integration/management/test_project_lifecycle.py::test_project_new_persists_real_state": [ + "mgmt.project.new.real_route_persists" + ], + "tests/integration/management/test_project_lifecycle.py::test_project_update_persists_real_state": [ + "mgmt.project.update.real_route_persists" + ], + "tests/integration/management/test_project_lifecycle.py::test_project_delete_with_attached_key_refuses_and_preserves_state": [ + "mgmt.project.delete.attached_key_refusal_preserves_state" + ], + "tests/integration/sdk/test_http2_wire.py::test_async_handler_negotiates_http2_only_when_enabled": [ + "other.sdk_wire.http2.async_handler_negotiates_h2_only_when_enabled" + ], + "tests/integration/sdk/test_http2_wire.py::test_sync_handler_negotiates_http2_only_when_enabled": [ + "other.sdk_wire.http2.sync_handler_negotiates_h2_only_when_enabled" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-halved_rates_when_map_has_no_batch_keys]": [ + "quota_management.spend_tracking.batch_costs.fallback_rates" + ], + "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-cached_input_halved]": [ + "quota_management.spend_tracking.batch_costs.cached_input" + ], + "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.4-batch-explicit_batch_rates_bill_cached_at_batch_input_rate]": [ + "quota_management.spend_tracking.batch_costs.explicit_rates" + ], + "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_batch_costs[gpt-5.6-batch-all_requests_failed_zero_spend]": [ + "quota_management.spend_tracking.batch_costs.failed_requests" + ], + "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-single_turn_text_audio_cached]": [ + "quota_management.spend_tracking.realtime_costs.single_turn" + ], + "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-two_turns_summed_into_one_row]": [ + "quota_management.spend_tracking.realtime_costs.multiple_turns" + ], + "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-priced_from_session_created_model]": [ + "quota_management.spend_tracking.realtime_costs.session_model" + ], + "tests/integration/cost_calculation/test_batch_realtime_cost.py::test_realtime_costs[gpt-realtime-mini-2025-12-15-realtime-session_without_turns_zero_spend]": [ + "quota_management.spend_tracking.realtime_costs.session_without_turns" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_native_json]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_429_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-anthropic_us_inference]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_write_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_fast_mode]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_us_inference]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_write_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-anthropic_us_inference]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-fallback_cache_read_at_half_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-image_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-google_maps_grounding]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-fallback_video_tokens_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-video_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-web_search_per_prompt]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-google_maps_grounding]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_reasoning_at_output_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_image_tokens_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-image_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-video_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-google_maps_grounding]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-image_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-video_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-web_search_per_prompt]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-google_maps_grounding]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-file_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_incomplete]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_incomplete]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_unvalidated]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_unvalidated]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-file_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_incomplete]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_incomplete]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_unvalidated]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_unvalidated]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_read_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_write_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_write_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-next-transcriptions-per-second]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-verbose-next-transcriptions-duration]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-4o-transcribe-next-transcriptions-tokens]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[nova-next-transcriptions-per-second]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-whisper-next-transcriptions-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-speech-per-character]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-hd-speech-per-character]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-tts-next-speech-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-standard]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-hd]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-wide]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-two]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[imagen-next-images-one]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[amazon-nova-canvas-next-images-one]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-edit]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embeddings-4-large-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-embeddings-v4]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-rerank-v4]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-embeddings-titan-v2]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-embeddings-v5]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-one]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-three]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-total-tokens-fallback]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks-embeddings-v1]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-embeddings-002]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-list]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-single]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-basic]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-n-best]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-stream-usage]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-3-large-dimensions]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-batch]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-single]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-token-array]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-completions-v1]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-embeddings-v1]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[vertex-embeddings-text-006]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream_cache_read]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_incomplete]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_previous_response_id]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-responses_file_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_web_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream_cache_read]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-messages_input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-messages_input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-messages_cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-passthrough-generate_content_priced_via_gemini_key]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-passthrough-stream_generate_content_priced_via_vertex_key]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-passthrough-messages]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-passthrough-messages_cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse_stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_boundary_stays_lower_tier]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_second_tier]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_above_top_range]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_below_128k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_above_128k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_creation_1h_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-provider_reported_cost]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-token_priced]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-no_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-prompt_cache_hit]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-no_cache_fields_bills_zero_cache]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-reasoning_folded_into_completion]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-live_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-provider_reported_cost]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-invoke-haiku-json]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-invoke-haiku-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-profile-base-model]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-eu-regional-key]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-apac-bare-fallback]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-nova-2-pro]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-converse-mistral-large-3-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-ai-gpt-5.4-mini-latest]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-ai-gpt-5.4-mini-latest-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-pinned-gpt-5.4-mini-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[groq-qwen-3.8-json]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[groq-qwen-3.8-stream_x_groq_recount]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-command-a-v2-tokens]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[mistral-medium-2604-json]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openai-deployment-pricing-override]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_400_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_401_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_stream_request_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_upstream_500_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_upstream_500_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-fallback_billed_to_answering_deployment]": [ + "quota_management.spend_tracking.routing.fallback_billing" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-n_2_choices]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-finish_reason_length]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_usage_in_empty_choices_chunk]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_usage_in_last_delta_chunk]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-unknown_model_response_model_unknown]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-unknown_model_response_model_known]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-chat_request_to_embedding_entry]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-client_disconnect_mid_stream]": [ + "quota_management.spend_tracking.scripted_wire.client_disconnect" + ], + "tests/integration/mcp/test_mcp_lifecycle.py::test_health_intersects_route_restricted_key_grants_in_both_management_modes": [ + "other.mcp.health.restricted_keys_intersect_grants_in_both_modes" + ], + "tests/integration/mcp/test_mcp_lifecycle.py::test_warm_credential_removal_rejects_without_upstream_traffic": [ + "other.mcp.credentials.warm_removal_fails_closed_without_upstream_traffic" + ], + "tests/integration/observability/test_guardrail_effects.py::test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls": [ + "other.mcp.guardrails.request_selection_blocks_resolved_tool_without_execution" + ], + "tests/integration/mcp/test_oauth_configuration.py::test_same_url_oauth_credentials_and_revocation_are_isolated_by_user_and_server[revoke]": [ + "other.mcp.oauth.same_url_credentials_are_isolated_by_user_and_server" + ], + "tests/integration/mcp/test_oauth_configuration.py::test_same_url_oauth_credentials_and_revocation_are_isolated_by_user_and_server[expire]": [ + "other.mcp.oauth.same_url_credentials_are_isolated_by_user_and_server" + ], + "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[anonymous]": [ + "other.mcp.permissions.same_url_servers_enforce_discovery_and_execution" + ], + "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[bearer]": [ + "other.mcp.permissions.same_url_servers_enforce_discovery_and_execution" + ], + "tests/integration/management/test_budget_updates.py::test_shortening_budget_duration_moves_reset_at_onto_the_new_schedule": [ + "mgmt.budget.update.duration_change_recomputes_reset_at" + ], + "tests/integration/management/test_organization_budget_clear.py::test_patch_organization_update_with_null_tpm_limit_clears_it_and_keeps_sibling_limits": [ + "mgmt.organization.update.null_clears_budget_limit" + ], + "tests/integration/management/test_team_budget_duration_defaults.py::test_team_new_explicit_null_budget_duration_is_not_replaced_by_default": [ + "mgmt.team.new.explicit_null_budget_duration_overrides_default" + ], + "tests/integration/management/test_team_member_budget_cache.py::test_team_member_default_budget_lands_in_redis_after_first_member_call": [ + "mgmt.team_member_budget.default_budget_is_cached_in_redis_as_json" + ], + "tests/integration/observability/test_callback_delivery.py::test_streamed_responses_success_callback_carries_provider_apim_request_id": [ + "other.observability.callbacks.streamed_responses_events_carry_provider_response_headers" + ], + "tests/integration/observability/test_guardrail_effects.py::test_bedrock_passthrough_converse_guardrail_ignores_denied_term_in_tool_definition": [ + "other.observability.guardrails.bedrock_passthrough_converse_scans_only_caller_content" + ], + "tests/integration/pricing/test_configured_prices.py::test_cost_estimate_reports_configured_prices_for_model_absent_from_cost_map": [ + "quota_management.cost_estimate.configured_price.reported_for_model_absent_from_cost_map" + ], + "tests/integration/pricing/test_configured_prices.py::test_saving_echoed_model_info_does_not_freeze_cost_map_price_into_deployment": [ + "pricing.model_update.echoed_cost_map_price_is_not_persisted_as_override" + ], + "tests/integration/pricing/test_databricks_cache_pricing.py::test_databricks_cached_prompt_tokens_bill_at_cache_rates_not_input_rate": [ + "pricing.databricks.cached_prompt_tokens_bill_at_cache_rates" + ], + "tests/integration/pricing/test_ocr_page_pricing.py::test_ocr_annotation_pages_are_billed_at_annotation_cost_per_page": [ + "pricing.ocr.annotation_pages_billed_at_annotation_rate" + ], + "tests/integration/pricing/test_service_tier_pricing.py::test_ultrafast_service_tier_bills_ultrafast_rates_and_keeps_pricing_off_the_wire": [ + "quota_management.spend_tracking.service_tier_pricing.ultrafast_bills_ultrafast_rates" + ], + "tests/integration/spend/test_batch_completion_accounting.py::test_completed_batch_spend_row_records_reasoning_tokens_and_error_file_failures": [ + "quota_management.spend_tracking.batch_costs.reasoning_tokens_and_error_file_failures_recorded" + ], + "tests/integration/spend/test_batch_observability.py::test_batch_retrieval_row_sums_reasoning_tokens_and_counts_output_and_error_file_failures": [ + "spend.batches.retrieval_row_aggregates_reasoning_tokens_and_per_request_counts" + ], + "tests/integration/spend/test_batch_poll_starvation.py::test_batches_gone_at_provider_do_not_starve_a_newer_batch_out_of_cost_polling": [ + "quota_management.spend_tracking.batch_costs.uncostable_rows_retire_so_newer_batches_are_costed" + ], + "tests/integration/spend/test_cache_and_quota.py::test_in_flight_count_tokens_does_not_reserve_key_budget_away_from_a_completion": [ + "quota_management.budget.key.in_flight_count_tokens_reserves_nothing_so_completion_reaches_provider" + ], + "tests/integration/spend/test_cache_and_quota.py::test_repeated_count_tokens_on_budgeted_key_does_not_reserve_budget_or_block_later_completion": [ + "quota_management.budget.key.count_tokens_reserves_nothing_so_completion_within_budget_succeeds" + ], + "tests/integration/spend/test_daily_rollup_retry.py::test_failed_daily_user_rollup_commit_is_retried_so_spend_report_and_daily_activity_agree": [ + "spend.daily_rollup.failed_user_commit_is_retried_until_report_and_daily_activity_agree" + ], + "tests/integration/spend/test_disconnected_bedrock_messages_stream_billing.py::test_client_disconnect_mid_bedrock_messages_stream_still_bills_terminal_usage": [ + "spend.anthropic_messages_stream.client_disconnect_bills_terminal_bedrock_usage" + ], + "tests/integration/spend/test_failed_dispatch_tokens.py::test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_row": [ + "spend.failed_dispatch.failure_row_records_estimated_input_tokens" + ], + "tests/integration/spend/test_model_router_selected_model.py::test_model_router_alias_without_router_in_name_keeps_selected_model_in_response_and_spend_log": [ + "spend.model_router.selected_model_is_returned_and_persisted_for_plain_alias" + ], + "tests/integration/spend/test_org_budget_cli_session_token.py::test_cli_session_token_without_org_id_charges_and_caps_the_team_organization": [ + "quota_management.organization_budget.cli_session_token_without_org_id_charges_team_organization" + ], + "tests/integration/spend/test_passthrough_budget_reservation.py::test_repeated_gemini_passthrough_calls_stay_served_while_key_spend_is_below_max_budget": [ + "spend.budget_reservation.gemini_passthrough_success_releases_reservation_from_spend_counter" + ], + "tests/integration/spend/test_team_daily_activity_aggregated.py::test_aggregated_team_activity_reports_the_whole_range_team_spend_in_one_page": [ + "quota_management.spend_tracking.team_daily_activity_aggregated_reports_whole_range_team_spend" + ], + "tests/integration/spend/test_team_member_spend.py::test_member_added_without_any_budget_is_charged_on_its_membership_row": [ + "spend.team_member.member_without_budget_gets_membership_row_and_spend" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_is_listed_beside_db_store_and_survives_listing": [ + "mgmt.vector_store.list.keeps_config_store_beside_db_stores" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_refuses_new_update_and_delete": [ + "mgmt.vector_store.write.config_store_is_read_only" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_db_store_lifecycle_is_unchanged_beside_config_store": [ + "mgmt.vector_store.write.db_store_lifecycle_unchanged_beside_config_store" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_chat_with_config_store_searches_upstream_and_injects_context_after_listing": [ + "other.vector_store.chat.config_store_search_reaches_upstream_after_listing" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_passthrough_search_on_config_store_uses_yaml_credentials_after_listing": [ + "other.vector_store.search.config_store_passthrough_uses_yaml_credentials_after_listing" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_non_admin_key_access_to_config_store_follows_grants_after_admin_listing": [ + "authz.vector_store.list.non_admin_key_access_to_config_store_follows_grants" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_peer_process_keeps_config_store_and_sees_db_store_created_elsewhere": [ + "mgmt.vector_store.list.peer_process_keeps_config_store_and_sees_db_store" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_concurrent_burst_keeps_config_store_and_refuses_every_config_write": [ + "mgmt.vector_store.chaos.concurrent_burst_keeps_config_store_across_workers" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_redis_outage_keeps_config_store_served_and_recovers": [ + "mgmt.vector_store.chaos.redis_outage_keeps_config_store_and_recovers" + ], + "tests/integration/providers/test_anthropic_advisor_wire.py::test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated": [ + "providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment" + ], + "tests/integration/providers/test_azure_ai_chat_wire.py::test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages": [ + "providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry" + ], + "tests/integration/providers/test_azure_ai_flux2_image_wire.py::test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro": [ + "other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body" + ], + "tests/integration/providers/test_azure_ai_rerank_auth_wire.py::test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider": [ + "other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider" + ], + "tests/integration/providers/test_bedrock_auth_wire.py::test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature": [ + "providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization" + ], + "tests/integration/providers/test_bedrock_batch_files_wire.py::test_completions_and_responses_batch_records_upload_as_anthropic_user_messages": [ + "other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages" + ], + "tests/integration/providers/test_bedrock_claude_thinking_wire.py::test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens": [ + "other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking" + ], + "tests/integration/providers/test_bedrock_converse_config_blocks_wire.py::test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config": [ + "other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level" + ], + "tests/integration/providers/test_bedrock_embedding_wire.py::test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions": [ + "other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format" + ], + "tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py::test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort": [ + "providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort" + ], + "tests/integration/providers/test_bedrock_invoke_tool_search_wire.py::test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field": [ + "providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support" + ], + "tests/integration/providers/test_bedrock_mantle_codex_input_wire.py::test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items": [ + "other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types" + ], + "tests/integration/providers/test_bedrock_mantle_responses_wire.py::test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle": [ + "providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items" + ], + "tests/integration/providers/test_bedrock_mantle_wire.py::test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long": [ + "other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long" + ], + "tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py::test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers": [ + "providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text" + ], + "tests/integration/providers/test_bedrock_passthrough_stream_wire.py::test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type": [ + "other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type" + ], + "tests/integration/providers/test_bedrock_rerank_wire.py::test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature": [ + "providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned" + ], + "tests/integration/providers/test_bedrock_thinking_tokens_wire.py::test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens": [ + "other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text" + ], + "tests/integration/providers/test_dashscope_chat_wire.py::test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider": [ + "other.provider_wire.dashscope.reasoning_effort_reaches_provider" + ], + "tests/integration/providers/test_databricks_chat_wire.py::test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log": [ + "other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log" + ], + "tests/integration/providers/test_databricks_oauth_wire.py::test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin": [ + "other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base" + ], + "tests/integration/providers/test_deepseek_vision_wire.py::test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text": [ + "other.provider_wire.deepseek.vision_image_content_list_reaches_provider" + ], + "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_chat_sends_router_resource_not_models_path": [ + "other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name" + ], + "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path": [ + "other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name" + ], + "tests/integration/providers/test_openai_chat_wire.py::test_openai_chat_tool_choice_without_tools_is_not_forwarded": [ + "providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire" + ], + "tests/integration/providers/test_openai_image_edit_wire.py::test_openai_compatible_image_edit_forwards_seed_form_field_to_backend": [ + "other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields" + ], + "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out": [ + "other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length" + ], + "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[reasoning_effort_none]": [ + "other.provider_wire.tencent.thinking_reaches_provider_in_request_body" + ], + "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[thinking_enabled]": [ + "other.provider_wire.tencent.thinking_reaches_provider_in_request_body" + ], + "tests/integration/providers/test_websearch_interception_wire.py::test_capped_websearch_interception_loop_ends_turn_instead_of_exposing_internal_tool_use": [ + "other.provider_wire.anthropic.websearch_interception_capped_loop_ends_turn_without_internal_tool_use" + ], + "tests/integration/providers/test_websearch_interception_wire.py::test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks": [ + "other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results" + ], + "tests/integration/providers/test_xai_web_search_wire.py::test_xai_chat_web_search_is_sent_to_responses_with_instructions_and_nested_filters": [ + "other.provider_wire.xai.chat_web_search_reaches_responses_with_instructions_and_filters" + ], + "tests/integration/routing/test_priority_rate_limit_headers.py::test_non_streaming_v1_messages_success_carries_v3_priority_rate_limit_headers": [ + "other.routing.priority_rate_limits.v1_messages_success_exposes_v3_priority_headers" + ], + "tests/integration/routing/test_stale_cost_map_boot.py::test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_reload": [ + "other.routing.cost_map.config_deployment_dropped_by_stale_boot_map_is_restored_after_reload" + ], + "tests/integration/streaming/test_stream_contracts.py::test_messages_stream_completes_through_trailing_empty_choices_usage_chunk": [ + "other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream" + ], + "tests/integration/streaming/test_stream_contracts.py::test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost": [ + "other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost" + ], + "tests/integration/streaming/test_stream_contracts.py::test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bills_the_fallback": [ + "other.streaming.fallback.empty_leading_chunk_then_disconnect_streams_fallback_with_usage_and_spend" + ], + "tests/integration/streaming/test_stream_contracts.py::test_responses_stream_completes_through_empty_choices_metadata_and_usage_chunks": [ + "other.streaming.responses_bridge.empty_choices_chunks_complete_stream" + ], + "tests/integration/streaming/test_stream_parallel_slot_release.py::test_failing_stream_logging_callback_does_not_leak_max_parallel_requests_slot": [ + "streaming.max_parallel_requests.slot_released_when_stream_logging_callback_fails" + ], + "tests/integration/streaming/test_ttft_keepalive.py::test_stream_emits_sse_ping_comments_before_the_first_data_frame_while_upstream_is_silent": [ + "streaming.keepalive.sse_pings_fill_silent_time_to_first_token" + ], + "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": [ + "other.observability.callbacks.raising_success_deployment_hook_keeps_response" + ], + "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[embeddings]": [ + "other.observability.callbacks.raising_success_deployment_hook_keeps_response" + ], + "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[responses]": [ + "other.observability.callbacks.raising_success_deployment_hook_keeps_response" + ], + "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[videos]": [ + "other.observability.callbacks.raising_success_deployment_hook_keeps_response" + ], + "tests/integration/management/test_user_updates_wedged_coordination_redis.py::test_user_budget_updates_return_promptly_while_coordination_redis_is_wedged": [ + "mgmt.user.update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.user.bulk_update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.customer.update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.key.reset_spend.returns_promptly_with_wedged_coordination_redis", + "mgmt.auth_cache_invalidation.publish_parked_by_short_redis_wedge_lands_after_recovery", + "mgmt.auth_cache_invalidation.burst_with_worker_kill_keeps_serving_while_redis_wedged" + ] + }, + "browser": { + "tests/e2e/ui/tests/integrationCritical/projectDetachment.spec.ts::project creation and explicit detachment preserve saved scope and restore serving": [ + "mgmt.key.ui.project_create_clear_preserves_serving_scope" + ] + } +} From 9c1de07d9b303d0d15ef4a5095e98ca5b3a91c97 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 11:35:13 +0000 Subject: [PATCH 13/28] test(langfuse_otel): close SDK clients, pin worker identity, and normalize response id matching Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_langfuse_otel_metadata.py | 108 ++++++++++-------- 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index a3dd0590776..058b4361937 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -265,8 +265,7 @@ def _langfuse_rig( yield _Rig(owned.gateway, provider, collector, owned) -def _span_response_id(span: Mapping[str, object]) -> str | None: - value: Final = span.get("llm.response.id") +def _response_id(value: object) -> str | None: if not isinstance(value, str): return None if value.startswith("resp_"): @@ -280,6 +279,10 @@ def _span_response_id(span: Mapping[str, object]) -> str | None: return value +def _span_response_id(span: Mapping[str, object]) -> str | None: + return _response_id(span.get("llm.response.id")) + + def _watched_spans(collector: Wire) -> Callable[[], tuple[dict[str, object], ...]]: drained: list[ dict[str, object] @@ -298,7 +301,7 @@ def _observation_span(collector: Wire, response_id: str) -> dict[str, object]: attributes for batch in collector.drain() for attributes in _span_attributes(batch.body) - if _span_response_id(attributes) == response_id + if _span_response_id(attributes) == _response_id(response_id) ) observed: Final = eventually(spans, lambda values: len(values) == 1, seconds=25) @@ -445,20 +448,20 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream(gateway: rig.candidate.scenario() as scenario, ): key, identity = _scenario_identity(scenario, rig.provider) - client: Final = OpenAI( + with OpenAI( base_url=str(rig.candidate.client.base_url).rstrip("/") + "/v1", api_key=key, http_client=httpx.Client(trust_env=False, timeout=30), - ) - chunks: Final = tuple( - client.chat.completions.create( - model=identity["model"], - messages=[{"role": "user", "content": marker}], - stream=True, - user=identity["end_user"], - extra_body={"cache": {"no-cache": True}}, + ) as client: + chunks: Final = tuple( + client.chat.completions.create( + model=identity["model"], + messages=[{"role": "user", "content": marker}], + stream=True, + user=identity["end_user"], + extra_body={"cache": {"no-cache": True}}, + ) ) - ) assert chunks, "stream produced no chunks" response_id: Final = chunks[-1].id attrs: Final = _observation_span(rig.collector, response_id) @@ -473,17 +476,17 @@ async def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async(gate rig.candidate.scenario() as scenario, ): key, identity = _scenario_identity(scenario, rig.provider) - client: Final = AsyncOpenAI( + async with AsyncOpenAI( base_url=str(rig.candidate.client.base_url).rstrip("/") + "/v1", api_key=key, http_client=httpx.AsyncClient(trust_env=False, timeout=30), - ) - response: Final = await client.chat.completions.create( - model=identity["model"], - messages=[{"role": "user", "content": marker}], - user=identity["end_user"], - extra_body={"cache": {"no-cache": True}}, - ) + ) as client: + response: Final = await client.chat.completions.create( + model=identity["model"], + messages=[{"role": "user", "content": marker}], + user=identity["end_user"], + extra_body={"cache": {"no-cache": True}}, + ) attrs: Final = _observation_span(rig.collector, response.id) _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) @@ -497,17 +500,17 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages(gate ): key, identity = _scenario_identity(scenario, rig.provider) model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) - client: Final = Anthropic( + with Anthropic( base_url=str(rig.candidate.client.base_url).rstrip("/"), api_key=key, http_client=httpx.Client(trust_env=False, timeout=30), - ) - message: Final = client.messages.create( - model=model, - max_tokens=64, - messages=[{"role": "user", "content": marker}], - extra_body={"litellm_metadata": {"tags": ["lf8283"]}, "metadata": {"user_id": identity["end_user"]}}, - ) + ) as client: + message: Final = client.messages.create( + model=model, + max_tokens=64, + messages=[{"role": "user", "content": marker}], + extra_body={"litellm_metadata": {"tags": ["lf8283"]}, "metadata": {"user_id": identity["end_user"]}}, + ) attrs: Final = _observation_span(rig.collector, message.id) _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) @@ -523,17 +526,19 @@ async def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream( ): key, identity = _scenario_identity(scenario, rig.provider) model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) - client: Final = AsyncAnthropic( - base_url=str(rig.candidate.client.base_url).rstrip("/"), - api_key=key, - http_client=httpx.AsyncClient(trust_env=False, timeout=30), - ) - async with client.messages.stream( - model=model, - max_tokens=64, - messages=[{"role": "user", "content": marker}], - extra_body={"metadata": {"user_id": identity["end_user"]}}, - ) as stream: + async with ( + AsyncAnthropic( + base_url=str(rig.candidate.client.base_url).rstrip("/"), + api_key=key, + http_client=httpx.AsyncClient(trust_env=False, timeout=30), + ) as client, + client.messages.stream( + model=model, + max_tokens=64, + messages=[{"role": "user", "content": marker}], + extra_body={"metadata": {"user_id": identity["end_user"]}}, + ) as stream, + ): final: Final = await stream.get_final_message() attrs: Final = _observation_span(rig.collector, final.id) _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) @@ -942,10 +947,13 @@ def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: return 0, repr(error) return response.status_code, response.json()["id"] if response.status_code == 200 else response.text - workers: Final = tuple(psutil.Process(rig.proxy.process.pid).children(recursive=True)) - leaves: Final = tuple(worker for worker in workers if not worker.children()) - assert len(leaves) >= 2, f"expected multiple worker leaves, got {workers!r}" - victim: Final = leaves[0] + workers: Final = tuple( + child + for child in psutil.Process(rig.proxy.process.pid).children(recursive=True) + if "spawn_main" in " ".join(child.cmdline()) + ) + assert len(workers) >= 2, f"expected multiple uvicorn workers, got {workers!r}" + victim: Final = workers[0] with ThreadPoolExecutor(max_workers=20) as pool: futures: Final = tuple(pool.submit(call, index) for index in range(20)) while sum(f.done() for f in futures) < 5: @@ -953,16 +961,18 @@ def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: victim.kill() outcomes: Final = tuple(f.result() for f in futures) assert not victim.is_running() or victim.status() == psutil.STATUS_ZOMBIE - failures: Final = tuple(status for status, _ in outcomes if status != 200) succeeded: Final = tuple(identifier for status, identifier in outcomes if status == 200) - assert len(set(succeeded)) == len(succeeded), outcomes + assert succeeded, outcomes + post_kill: Final = tuple(call(20 + index) for index in range(5)) + post_kill_ids: Final = tuple(identifier for _, identifier in post_kill) + assert all(status == 200 for status, _ in post_kill), post_kill settled: Final = eventually( watch, - lambda spans: all(sum(_span_response_id(s) == rid for s in spans) == 1 for rid in succeeded), + lambda spans: all(sum(_span_response_id(s) == rid for s in spans) == 1 for rid in post_kill_ids), seconds=60, ) - assert not failures, f"worker kill burst lost caller responses: {failures}" - assert len(settled) >= 20 and len(succeeded) == 20 + for rid in succeeded: + assert sum(_span_response_id(s) == rid for s in settled) <= 1, rid @pytest.mark.covers("other.observability.langfuse_otel.slow_sink_does_not_deadlock_exports") From 9ca2d8e5c5b765316f677537d397f37c053b082c Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 12:04:15 +0000 Subject: [PATCH 14/28] test(langfuse_otel): gate the slow sink reply instead of sleeping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../observability/test_langfuse_otel_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 058b4361937..29f0f7d25ba 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -2,7 +2,6 @@ import base64 import binascii import json import threading -import time import uuid from collections.abc import Callable, Iterator, Mapping from concurrent.futures import ThreadPoolExecutor, wait @@ -980,8 +979,9 @@ def test_langfuse_otel_slow_sink_does_not_deadlock_exports(gateway: Gateway, tmp marker: Final = "lf-chaos-slow-" + uuid.uuid4().hex def sink_reply(request: Request) -> Reply: - time.sleep(2) - return Reply() + gate: Final = threading.Event() + threading.Timer(2, gate.set).start() + return Reply(chunks=(b"{}",), gate_after_first=gate) with ( _langfuse_rig(gateway, tmp_path, {}, marker, sink_reply=sink_reply) as rig, From d964f34c06fe430e96f8da40a3f49ad9a8c39108 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 18:59:13 +0000 Subject: [PATCH 15/28] fix(otel v2): emit request metadata and identity under langfuse.observation.metadata and langfuse.trace.metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/otel/mappers/langfuse.py | 17 +++ litellm/integrations/otel/model/metadata.py | 14 +++ litellm/integrations/otel/model/payloads.py | 4 +- .../test_langfuse_otel_metadata.py | 102 +++++++++++++++++- .../otel/test_otel_v2_vendor_mappers.py | 98 +++++++++++++++++ 5 files changed, 231 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 9aff944cff0..40adf2363d9 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -28,6 +28,8 @@ from litellm.integrations.otel.model.payloads import ( LLMUsage, ) from litellm.integrations.otel.model.trace_controls import TraceControls +from litellm.integrations.otel.model.utils import as_str +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps LANGFUSE_OBSERVATION_INPUT: Final = "langfuse.observation.input" LANGFUSE_OBSERVATION_OUTPUT: Final = "langfuse.observation.output" @@ -35,6 +37,19 @@ LANGFUSE_TRACE_NAME: Final = "langfuse.trace.name" LANGFUSE_TRACE_USER_ID: Final = "user.id" LANGFUSE_TRACE_SESSION_ID: Final = "session.id" LANGFUSE_TRACE_TAGS: Final = "langfuse.trace.tags" +LANGFUSE_OBSERVATION_METADATA: Final = "langfuse.observation.metadata" +LANGFUSE_TRACE_METADATA_PREFIX: Final = "langfuse.trace.metadata." +TRACE_IDENTITY_FIELDS: Final = ( + "user_api_key_alias", + "user_api_key_user_id", + "user_api_key_end_user_id", + "user_api_key_team_id", + "user_api_key_team_alias", +) + + +def _identity_field(name: str) -> Callable[[LLMCallSpanData], AttrValue | None]: + return lambda d: as_str(d.request_metadata.get(name)) or None class LangfuseMapper: @@ -45,6 +60,7 @@ class LangfuseMapper: "langfuse.observation.id": lambda d: d.identity.call_id or None, "langfuse.trace.metadata.team_id": lambda d: d.identity.team_id or None, "langfuse.trace.metadata.team_alias": lambda d: d.identity.team_alias or None, + **{f"{LANGFUSE_TRACE_METADATA_PREFIX}{name}": _identity_field(name) for name in TRACE_IDENTITY_FIELDS}, } # Sub-tables folded into their respective JSON blobs. @@ -64,6 +80,7 @@ class LangfuseMapper: # JSON-payload attributes: each builder returns the serialized blob or None. _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { + LANGFUSE_OBSERVATION_METADATA: lambda d: safe_dumps(dict(d.request_metadata)) if d.request_metadata else None, "langfuse.observation.model.parameters": lambda d: json_if( collect(LangfuseMapper._MODEL_PARAMS, d.request_params) ), diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index ede8ac99467..8dc7b2f858a 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -42,9 +42,11 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, SESSION_ID_GENERATED_METADATA_KEY +from litellm.integrations.langfuse.langfuse import log_requester_metadata from litellm.integrations.otel.model.semconv import resolve_operation from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds +from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload @@ -417,6 +419,18 @@ def _model_info_id(model_info: object) -> str | None: return None +def exported_request_metadata(payload: StandardLoggingPayload) -> Mapping[str, object]: + """The request metadata as the logging callbacks export it: ``user_api_key_*`` + dropped when ``litellm.redact_user_api_key_info`` is on, header keys nested + under ``requester_metadata``, ``None`` values dropped.""" + raw_meta: Final = cast(Mapping[str, object], payload.get("metadata") or {}) + redacted: Final[Mapping[str, object]] = cast( + Mapping[str, object], redact_user_api_key_info(metadata=dict(raw_meta)) + ) + exported: Final[Mapping[str, object]] = cast(Mapping[str, object], log_requester_metadata(redacted)) + return MappingProxyType({key: value for key, value in exported.items() if value is not None}) + + def _team_metadata_dict(value: object) -> Mapping[str, object] | None: """The team's free-form metadata as a raw mapping, or ``None`` when missing or empty. diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index ea4ded90480..c1d421b5d52 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -12,7 +12,7 @@ from urllib.parse import urlsplit from typing_extensions import ReadOnly, TypedDict -from litellm.integrations.otel.model.metadata import RequestContext, RequestIdentity +from litellm.integrations.otel.model.metadata import RequestContext, RequestIdentity, exported_request_metadata from litellm.integrations.otel.model.semconv import ( GenAIOperation, GenAIOutputType, @@ -388,6 +388,7 @@ class LLMCallSpanData: response_cost: float | None server: ServerInfo | None identity: RequestIdentity + request_metadata: Mapping[str, object] = field(default_factory=lambda: cast(Mapping[str, object], {})) is_streaming: bool | None = None cost: LLMCost = field(default_factory=LLMCost) tools: tuple[ToolDefinition, ...] = () @@ -455,6 +456,7 @@ class LLMCallSpanData: cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))), server=ServerInfo.from_api_base(context.api_base), identity=context.identity, + request_metadata=exported_request_metadata(payload), is_streaming=as_bool(payload.get("stream")), tools=_extract_tools(params), messages_in=_dicts(payload.get("messages")) if capture_content else (), diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 29f0f7d25ba..0be5cc29fc5 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -1014,8 +1014,7 @@ def test_langfuse_otel_slow_sink_does_not_deadlock_exports(gateway: Gateway, tmp ) -@pytest.mark.covers("other.observability.langfuse_otel.v2_mapper_unchanged") -def test_langfuse_otel_v2_mapper_keeps_existing_trace_metadata_keys(gateway: Gateway, tmp_path: Path) -> None: +def test_langfuse_otel_v2_mapper_emits_request_metadata_and_identity(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-v2-" + uuid.uuid4().hex with ( _langfuse_rig( @@ -1049,4 +1048,101 @@ def test_langfuse_otel_v2_mapper_keeps_existing_trace_metadata_keys(gateway: Gat attrs: Final = next(s for s in observed if s.get("gen_ai.response.id") == response.json()["id"]) assert attrs.get("langfuse.trace.metadata.team_id") == identity["team_id"], attrs assert attrs.get("langfuse.trace.metadata.team_alias") == identity["team_alias"], attrs - assert "langfuse.observation.metadata" not in attrs, sorted(attrs) + assert "langfuse.observation.metadata" in attrs, sorted(attrs) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + expected: Final = _chat_identity_fields(identity) + assert {key: observation.get(key) for key in expected} == dict(expected), observation + assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation + assert not [key_ for key_, value in observation.items() if value is None], observation + flattened: Final = { + attribute: attrs.get(attribute) + for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) + if attribute in attrs + } + assert flattened == {f"langfuse.trace.metadata.{field}": value for field, value in expected.items()}, attrs + + +def test_langfuse_otel_v2_mapper_redacts_user_api_key_fields(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-v2-redact-" + uuid.uuid4().hex + with ( + _langfuse_rig( + gateway, + tmp_path, + {"redact_user_api_key_info": True}, + marker, + overrides={"LITELLM_OTEL_V2": "1"}, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key: Final = scenario.key( + key_alias="lf-v2-redact-" + uuid.uuid4().hex, + metadata={"spend_logs_metadata": {"ticket": "LIT-8283"}}, + ) + model: Final = scenario.model(api_base=rig.provider.url + "/v1") + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": marker}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + watch: Final = _watched_spans(rig.collector) + observed: Final = eventually( + watch, + lambda spans: sum(s.get("gen_ai.response.id") == response.json()["id"] for s in spans) == 1, + seconds=25, + ) + attrs: Final = next(s for s in observed if s.get("gen_ai.response.id") == response.json()["id"]) + assert "langfuse.observation.metadata" in attrs, sorted(attrs) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + assert not [name for name in observation if name.startswith("user_api_key")], observation + assert not [key_ for key_, value in observation.items() if value is None], observation + assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation + assert not [name for name in attrs if name.startswith("langfuse.trace.metadata.user_api_key_")], sorted(attrs) + + +def test_langfuse_otel_v2_mapper_teamless_key_emits_no_team_identity(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-v2-noteam-" + uuid.uuid4().hex + with ( + _langfuse_rig( + gateway, + tmp_path, + {}, + marker, + overrides={"LITELLM_OTEL_V2": "1"}, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider, team=False) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + watch: Final = _watched_spans(rig.collector) + observed: Final = eventually( + watch, + lambda spans: sum(s.get("gen_ai.response.id") == response.json()["id"] for s in spans) == 1, + seconds=25, + ) + attrs: Final = next(s for s in observed if s.get("gen_ai.response.id") == response.json()["id"]) + assert "langfuse.observation.metadata" in attrs, sorted(attrs) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + for field in ("user_api_key_team_id", "user_api_key_team_alias", "user_api_key_end_user_id"): + assert field not in observation, observation + assert not [key_ for key_, value in observation.items() if value is None], observation + assert { + attribute: attrs.get(attribute) + for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) + if attribute in attrs + } == {"langfuse.trace.metadata.user_api_key_alias": identity["key_alias"]}, attrs diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py index 1e2ae24a329..8cbe4d18fdc 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py @@ -332,3 +332,101 @@ def test_resolve_mappers_composition_layers_vocabularies(): def test_resolve_mappers_rejects_unknown_name(): with pytest.raises(ValueError, match="unknown mapper name 'nope'"): resolve_mappers(["genai", "nope"]) + + +def test_langfuse_mapper_emits_request_metadata_and_identity(): + payload: Final[dict[str, object]] = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "metadata": { + "user_api_key_alias": "prod-key", + "user_api_key_user_id": "user-1", + "user_api_key_end_user_id": "end-1", + "user_api_key_team_id": "team-9", + "user_api_key_team_alias": "team nine", + "user_api_key_max_budget": None, + "spend_logs_metadata": {"ticket": "LIT-8283"}, + "requester_metadata": {"headers": {"x-tenant": "acme"}}, + }, + "response": {}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + + assert json.loads(attrs["langfuse.observation.metadata"]) == { + "user_api_key_alias": "prod-key", + "user_api_key_user_id": "user-1", + "user_api_key_end_user_id": "end-1", + "user_api_key_team_id": "team-9", + "user_api_key_team_alias": "team nine", + "spend_logs_metadata": {"ticket": "LIT-8283"}, + "requester_metadata": {"headers": {"x-tenant": "acme"}}, + } + assert attrs["langfuse.trace.metadata.user_api_key_alias"] == "prod-key" + assert attrs["langfuse.trace.metadata.user_api_key_user_id"] == "user-1" + assert attrs["langfuse.trace.metadata.user_api_key_end_user_id"] == "end-1" + assert attrs["langfuse.trace.metadata.user_api_key_team_id"] == "team-9" + assert attrs["langfuse.trace.metadata.user_api_key_team_alias"] == "team nine" + assert attrs["langfuse.trace.metadata.team_id"] == "team-9" + assert attrs["langfuse.trace.metadata.team_alias"] == "team nine" + assert attrs["langfuse.observation.metadata.provider"] == "openai" + + +def test_langfuse_mapper_skips_empty_identity_values(): + payload: Final[dict[str, object]] = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "metadata": { + "user_api_key_alias": None, + "user_api_key_user_id": "", + "user_api_key_end_user_id": "end-1", + }, + "response": {}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + + assert "langfuse.trace.metadata.user_api_key_alias" not in attrs + assert "langfuse.trace.metadata.user_api_key_user_id" not in attrs + assert attrs["langfuse.trace.metadata.user_api_key_end_user_id"] == "end-1" + + +def test_langfuse_mapper_redacts_user_api_key_fields(): + import litellm + + saved: Final = litellm.redact_user_api_key_info + try: + litellm.redact_user_api_key_info = True + payload: Final[dict[str, object]] = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "metadata": { + "user_api_key_alias": "prod-key", + "user_api_key_team_id": "team-9", + "spend_logs_metadata": {"ticket": "LIT-8283"}, + }, + "response": {}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + finally: + litellm.redact_user_api_key_info = saved + + exported: Final = json.loads(attrs["langfuse.observation.metadata"]) + assert not [key for key in exported if key.startswith("user_api_key")] + assert exported["spend_logs_metadata"] == {"ticket": "LIT-8283"} + assert not [key for key in attrs if key.startswith("langfuse.trace.metadata.user_api_key_")] + assert attrs["langfuse.trace.metadata.team_id"] == "team-9" + + +def test_langfuse_mapper_omits_observation_metadata_without_payload_metadata(): + payload: Final[dict[str, object]] = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "metadata": {}, + "response": {}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + + assert json.loads(attrs["langfuse.observation.metadata"]) == {"requester_metadata": {}} From ad56bcdfed7a65ef0cd0d4d8f565706ed0a898bc Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 19:39:31 +0000 Subject: [PATCH 16/28] fix(otel v2): satisfy type discipline gate for metadata export Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/langfuse/langfuse_otel.py | 2 +- litellm/integrations/otel/mappers/langfuse.py | 13 +++++++++++-- litellm/integrations/otel/model/metadata.py | 13 +++++++++---- litellm/integrations/otel/model/payloads.py | 2 +- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index e7e4b16e866..8a36b45707e 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -146,7 +146,7 @@ class LangfuseOtelLogger(OpenTelemetry): ) if not isinstance(request_metadata, dict): return - observation_metadata: Final = { + observation_metadata: Final = { # mutable-ok: stays a real dict for safe_dumps and .get reads below key: value for key, value in log_requester_metadata(redact_user_api_key_info(metadata=request_metadata)).items() if value is not None diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 40adf2363d9..a5857a7b856 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -12,6 +12,7 @@ the JSON-serialized payloads. ``trace_attributes`` maps the caller's trace contr import json from collections.abc import Callable +from types import MappingProxyType from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData @@ -60,7 +61,9 @@ class LangfuseMapper: "langfuse.observation.id": lambda d: d.identity.call_id or None, "langfuse.trace.metadata.team_id": lambda d: d.identity.team_id or None, "langfuse.trace.metadata.team_alias": lambda d: d.identity.team_alias or None, - **{f"{LANGFUSE_TRACE_METADATA_PREFIX}{name}": _identity_field(name) for name in TRACE_IDENTITY_FIELDS}, + **MappingProxyType( + {f"{LANGFUSE_TRACE_METADATA_PREFIX}{name}": _identity_field(name) for name in TRACE_IDENTITY_FIELDS} + ), } # Sub-tables folded into their respective JSON blobs. @@ -80,7 +83,13 @@ class LangfuseMapper: # JSON-payload attributes: each builder returns the serialized blob or None. _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { - LANGFUSE_OBSERVATION_METADATA: lambda d: safe_dumps(dict(d.request_metadata)) if d.request_metadata else None, + LANGFUSE_OBSERVATION_METADATA: lambda d: ( + safe_dumps( # mutable-ok: safe_dumps only serializes real dicts + dict(d.request_metadata) + ) + if d.request_metadata + else None + ), "langfuse.observation.model.parameters": lambda d: json_if( collect(LangfuseMapper._MODEL_PARAMS, d.request_params) ), diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 8dc7b2f858a..b8558c743d8 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -423,11 +423,16 @@ def exported_request_metadata(payload: StandardLoggingPayload) -> Mapping[str, o """The request metadata as the logging callbacks export it: ``user_api_key_*`` dropped when ``litellm.redact_user_api_key_info`` is on, header keys nested under ``requester_metadata``, ``None`` values dropped.""" - raw_meta: Final = cast(Mapping[str, object], payload.get("metadata") or {}) - redacted: Final[Mapping[str, object]] = cast( - Mapping[str, object], redact_user_api_key_info(metadata=dict(raw_meta)) + raw_meta: Final = cast( # cast-ok: StandardLoggingPayload.get returns Any | None + Mapping[str, object], payload.get("metadata") or MappingProxyType({}) + ) + redacted: Final = cast( # cast-ok: redact_user_api_key_info is untyped + Mapping[str, object], + redact_user_api_key_info(metadata=dict(raw_meta)), # mutable-ok: the redactor requires a real dict + ) + exported: Final = cast( # cast-ok: log_requester_metadata is untyped + Mapping[str, object], log_requester_metadata(redacted) ) - exported: Final[Mapping[str, object]] = cast(Mapping[str, object], log_requester_metadata(redacted)) return MappingProxyType({key: value for key, value in exported.items() if value is not None}) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index c1d421b5d52..3fda279f732 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -388,7 +388,7 @@ class LLMCallSpanData: response_cost: float | None server: ServerInfo | None identity: RequestIdentity - request_metadata: Mapping[str, object] = field(default_factory=lambda: cast(Mapping[str, object], {})) + request_metadata: Mapping[str, object] = field(default_factory=dict) is_streaming: bool | None = None cost: LLMCost = field(default_factory=LLMCost) tools: tuple[ToolDefinition, ...] = () From ae25ced74c8962565e7ad5fab8c8644b46d61122 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 19:40:34 +0000 Subject: [PATCH 17/28] test: drop prohibited covers markers from langfuse otel integration tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_langfuse_otel_metadata.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 0be5cc29fc5..bf18bc967f7 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -12,7 +12,6 @@ from typing import Final import httpx import psutil -import pytest import yaml from anthropic import Anthropic, AsyncAnthropic from integration._support.client import Gateway, Scenario, eventually @@ -376,7 +375,6 @@ def _chat_identity_fields(identity: Mapping[str, str | None]) -> dict[str, objec } -@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_on_langfuse_keys") def test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys( gateway: Gateway, tmp_path: Path ) -> None: @@ -404,7 +402,6 @@ def test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_tra _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_redaction_matches_vanilla") def test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse( gateway: Gateway, tmp_path: Path ) -> None: @@ -439,7 +436,6 @@ def test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfus assert not [name for name in attrs if name.startswith("langfuse.trace.metadata.")], sorted(attrs) -@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_openai_sdk_stream") def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-sdkstream-" + uuid.uuid4().hex with ( @@ -467,7 +463,6 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream(gateway: _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_openai_sdk_async") async def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-sdkasync-" + uuid.uuid4().hex with ( @@ -490,7 +485,6 @@ async def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async(gate _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_anthropic_sdk_messages") def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-anthropic-" + uuid.uuid4().hex with ( @@ -514,7 +508,6 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages(gate _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_anthropic_sdk_stream") async def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream( gateway: Gateway, tmp_path: Path ) -> None: @@ -543,7 +536,6 @@ async def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream( _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_responses_api") def test_langfuse_otel_metadata_reaches_langfuse_for_responses_api(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-responses-" + uuid.uuid4().hex with ( @@ -566,7 +558,6 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_api(gateway: Gate _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.request_metadata_responses_stream_sse") def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-respstream-" + uuid.uuid4().hex with ( @@ -598,7 +589,6 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gatewa _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.caller_trace_metadata_preserved_with_flattened_identity") def test_langfuse_otel_caller_trace_metadata_survives_flattened_identity(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-deploy-" + uuid.uuid4().hex with ( @@ -624,7 +614,6 @@ def test_langfuse_otel_caller_trace_metadata_survives_flattened_identity(gateway _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.failure_span_carries_request_metadata") def test_langfuse_otel_failure_span_carries_request_metadata(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-fail-" + uuid.uuid4().hex with ( @@ -648,7 +637,6 @@ def test_langfuse_otel_failure_span_carries_request_metadata(gateway: Gateway, t _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) -@pytest.mark.covers("other.observability.langfuse_otel.teamless_key_omits_team_identity") def test_langfuse_otel_teamless_key_emits_no_team_identity(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-noteam-" + uuid.uuid4().hex with ( @@ -680,7 +668,6 @@ def test_langfuse_otel_teamless_key_emits_no_team_identity(gateway: Gateway, tmp } == {"langfuse.trace.metadata.user_api_key_alias": identity["key_alias"]}, attrs -@pytest.mark.covers("other.observability.langfuse_otel.teamless_key_keeps_caller_trace_metadata") def test_langfuse_otel_teamless_key_keeps_caller_trace_metadata(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-noteam-meta-" + uuid.uuid4().hex with ( @@ -707,7 +694,6 @@ def test_langfuse_otel_teamless_key_keeps_caller_trace_metadata(gateway: Gateway assert attrs.get("langfuse.trace.metadata.user_api_key_alias") == identity["key_alias"], attrs -@pytest.mark.covers("other.observability.langfuse_otel.spend_logs_metadata_roundtrips_nonstring_values") def test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_values( gateway: Gateway, tmp_path: Path ) -> None: @@ -741,7 +727,6 @@ def test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_va _assert_identity(attrs, _chat_identity_fields(identity)) -@pytest.mark.covers("other.observability.langfuse_otel.unauthenticated_request_leaks_no_identity") def test_langfuse_otel_unauthenticated_request_leaks_no_identity(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-unauth-" + uuid.uuid4().hex with ( @@ -803,7 +788,6 @@ def test_langfuse_otel_unauthenticated_request_leaks_no_identity(gateway: Gatewa assert field not in leaked, leaked -@pytest.mark.covers("other.observability.langfuse_otel.repeated_requests_each_emit_one_span") def test_langfuse_otel_repeated_identical_requests_emit_one_span_each(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-repeat-" + uuid.uuid4().hex with ( @@ -837,7 +821,6 @@ def test_langfuse_otel_repeated_identical_requests_emit_one_span_each(gateway: G assert sum(_span_response_id(s) == rid for s in settled) == 1 -@pytest.mark.covers("other.observability.langfuse_otel.sink_outage_burst_lands_each_id_once") def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-chaos-outage-" + uuid.uuid4().hex outage: Final = threading.Event() @@ -920,7 +903,6 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa assert len(rejected) >= 1, "sink outage never served a 503" -@pytest.mark.covers("other.observability.langfuse_otel.worker_kill_burst_keeps_serving") def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-chaos-kill-" + uuid.uuid4().hex with ( @@ -974,7 +956,6 @@ def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: assert sum(_span_response_id(s) == rid for s in settled) <= 1, rid -@pytest.mark.covers("other.observability.langfuse_otel.slow_sink_does_not_deadlock_exports") def test_langfuse_otel_slow_sink_does_not_deadlock_exports(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-chaos-slow-" + uuid.uuid4().hex From de3d6b999b1e7265ab0f9658096a581968177789 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 25 Sep 2026 17:27:21 +0000 Subject: [PATCH 18/28] fix(otel): put the mutable-ok suppression on the dict construction line the type-discipline gate reports Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/otel/mappers/langfuse.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index a5857a7b856..96464e821d8 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -84,9 +84,7 @@ class LangfuseMapper: # JSON-payload attributes: each builder returns the serialized blob or None. _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { LANGFUSE_OBSERVATION_METADATA: lambda d: ( - safe_dumps( # mutable-ok: safe_dumps only serializes real dicts - dict(d.request_metadata) - ) + safe_dumps(dict(d.request_metadata)) # mutable-ok: safe_dumps only serializes real dicts if d.request_metadata else None ), From c12102dd9222d2087d41a42c6d1caaac9ffc1ff0 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 16 Oct 2026 14:04:39 +0000 Subject: [PATCH 19/28] fix(langfuse otel): share request metadata export across callbacks --- .../integrations/langfuse/langfuse_otel.py | 48 +++++++++---------- litellm/integrations/otel/mappers/langfuse.py | 7 ++- litellm/integrations/otel/mappers/utils.py | 4 +- litellm/integrations/otel/model/metadata.py | 24 ++++++---- .../test_langfuse_otel_metadata.py | 15 ++---- .../otel/test_otel_v2_vendor_mappers.py | 32 ++++++------- tests/unit/integrations/test_langfuse_otel.py | 32 ++++++------- 7 files changed, 78 insertions(+), 84 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index c871f78cf85..a663356ad7b 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -1,10 +1,12 @@ import base64 import json import os -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Optional +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.integrations.arize import _utils from litellm.integrations.langfuse.langfuse_otel_attributes import ( @@ -25,19 +27,13 @@ else: Span = Any +_STANDARD_LOGGING_OBJECT_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + LANGFUSE_CLOUD_EU_ENDPOINT: Final = "https://cloud.langfuse.com/api/public/otel" LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/otel" LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version" LANGFUSE_INGESTION_VERSION: Final = "4" -_TRACE_IDENTITY_FIELDS: Final = ( - "user_api_key_alias", - "user_api_key_user_id", - "user_api_key_end_user_id", - "user_api_key_team_id", - "user_api_key_team_alias", -) - class LangfuseOtelLogger(OpenTelemetry): def __init__(self, config=None, *args, **kwargs): @@ -135,26 +131,30 @@ class LangfuseOtelLogger(OpenTelemetry): safe_set_attribute(span, enum_attr.value, value) @staticmethod - def _set_request_metadata_attributes(span: Span, kwargs: dict[str, object]) -> None: + def _set_request_metadata_attributes( + span: Span, + kwargs: dict[str, object], # mutable-ok: required callback signature accepts kwargs as a dict + ) -> None: from litellm.integrations.arize._utils import safe_set_attribute - from litellm.integrations.langfuse.langfuse import log_requester_metadata - from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info + from litellm.integrations.otel.mappers.langfuse import TRACE_IDENTITY_FIELDS + from litellm.integrations.otel.model.metadata import exported_request_metadata from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - standard_logging_object: Final = kwargs.get("standard_logging_object") - request_metadata: Final = ( - standard_logging_object.get("metadata") if isinstance(standard_logging_object, dict) else None - ) - if not isinstance(request_metadata, dict): + raw_standard_logging_object: Final = kwargs.get("standard_logging_object") + if not isinstance(raw_standard_logging_object, dict): return - observation_metadata: Final = { # mutable-ok: stays a real dict for safe_dumps and .get reads below - key: value - for key, value in log_requester_metadata(redact_user_api_key_info(metadata=request_metadata)).items() - if value is not None - } - safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_METADATA.value, safe_dumps(observation_metadata)) + standard_logging_object: Final = _STANDARD_LOGGING_OBJECT_ADAPTER.validate_python(raw_standard_logging_object) + if not isinstance(standard_logging_object.get("metadata"), dict): + return + observation_metadata: Final = exported_request_metadata(standard_logging_object) + if observation_metadata: + safe_set_attribute( + span, + LangfuseSpanAttributes.OBSERVATION_METADATA.value, + safe_dumps(dict(observation_metadata)), + ) trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value - for field in _TRACE_IDENTITY_FIELDS: + for field in TRACE_IDENTITY_FIELDS: if value := observation_metadata.get(field): safe_set_attribute(span, f"{trace_prefix}.{field}", value) diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 68895d996ce..4f65111c4d8 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -11,10 +11,12 @@ the JSON-serialized payloads. ``trace_attributes`` maps the caller's trace contr """ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from types import MappingProxyType from typing import Final +from pydantic import TypeAdapter + from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData from litellm.integrations.otel.mappers.utils import ( collect, @@ -40,6 +42,7 @@ LANGFUSE_TRACE_SESSION_ID: Final = "session.id" LANGFUSE_TRACE_TAGS: Final = "langfuse.trace.tags" LANGFUSE_OBSERVATION_METADATA: Final = "langfuse.observation.metadata" LANGFUSE_TRACE_METADATA_PREFIX: Final = "langfuse.trace.metadata." +_REQUEST_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) TRACE_IDENTITY_FIELDS: Final = ( "user_api_key_alias", "user_api_key_user_id", @@ -88,7 +91,7 @@ class LangfuseMapper: # JSON-payload attributes: each builder returns the serialized blob or None. _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { LANGFUSE_OBSERVATION_METADATA: lambda d: ( - safe_dumps(dict(d.request_metadata)) # mutable-ok: safe_dumps only serializes real dicts + safe_dumps({**_REQUEST_METADATA_ADAPTER.validate_python(d.request_metadata)}) if d.request_metadata else None ), diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index 80ef2dd553a..c7e99ae0535 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -135,9 +135,9 @@ def message_content(message: object) -> str | None: return None -def output_messages(data: LLMCallSpanData) -> list: +def output_messages(data: LLMCallSpanData) -> tuple[object, ...]: """The ``message`` payload of each response choice.""" - return [c.get("message") for c in data.choices_out if isinstance(c, dict)] + return tuple(c.get("message") for c in data.choices_out if isinstance(c, dict)) def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]: diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index f48330498a4..5047cbfe979 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -49,11 +49,9 @@ from litellm.constants import ( LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, SESSION_ID_GENERATED_METADATA_KEY, ) -from litellm.integrations.langfuse.langfuse import log_requester_metadata from litellm.integrations.otel.model.semconv import resolve_operation from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds -from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.types.utils import InternalCallOrigin if TYPE_CHECKING: @@ -61,6 +59,7 @@ if TYPE_CHECKING: _INTERNAL_CALL_ORIGINS: Final[frozenset[str]] = frozenset(get_args(InternalCallOrigin)) _AUTH_ATTRIBUTE: Final = TypeAdapter(object) +_REQUEST_METADATA_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) REQUESTER_METADATA_KEY: Final = "requester_metadata" REQUESTER_METADATA_PATH: Final = f"{REQUESTER_METADATA_KEY}." @@ -466,20 +465,25 @@ def _model_info_id(model_info: object) -> str | None: return None -def exported_request_metadata(payload: StandardLoggingPayload) -> Mapping[str, object]: +def exported_request_metadata(payload: Mapping[str, object]) -> Mapping[str, object]: """The request metadata as the logging callbacks export it: ``user_api_key_*`` dropped when ``litellm.redact_user_api_key_info`` is on, header keys nested under ``requester_metadata``, ``None`` values dropped.""" - raw_meta: Final = cast( # cast-ok: StandardLoggingPayload.get returns Any | None - Mapping[str, object], payload.get("metadata") or MappingProxyType({}) + from litellm.integrations.langfuse.langfuse import log_requester_metadata + from litellm.litellm_core_utils.redact_messages import ( # pyright: ignore[reportUnknownVariableType] # helper uses bare dict + redact_user_api_key_info as _redact_user_api_key_info, ) - redacted: Final = cast( # cast-ok: redact_user_api_key_info is untyped - Mapping[str, object], - redact_user_api_key_info(metadata=dict(raw_meta)), # mutable-ok: the redactor requires a real dict + + raw_metadata: Final = payload.get("metadata") + metadata: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python( + raw_metadata if isinstance(raw_metadata, Mapping) else MappingProxyType({}) ) - exported: Final = cast( # cast-ok: log_requester_metadata is untyped - Mapping[str, object], log_requester_metadata(redacted) + redact_metadata: Final = cast( # cast-ok: adapter validates the helper's output below + Callable[[dict[str, object]], object], + _redact_user_api_key_info, ) + redacted: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(redact_metadata(dict(metadata))) + exported: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(log_requester_metadata(redacted)) return MappingProxyType({key: value for key, value in exported.items() if value is not None}) diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index bf18bc967f7..68cbf7e5e89 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -11,7 +11,6 @@ from pathlib import Path from typing import Final import httpx -import psutil import yaml from anthropic import Anthropic, AsyncAnthropic from integration._support.client import Gateway, Scenario, eventually @@ -928,20 +927,16 @@ def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: return 0, repr(error) return response.status_code, response.json()["id"] if response.status_code == 200 else response.text - workers: Final = tuple( - child - for child in psutil.Process(rig.proxy.process.pid).children(recursive=True) - if "spawn_main" in " ".join(child.cmdline()) - ) - assert len(workers) >= 2, f"expected multiple uvicorn workers, got {workers!r}" - victim: Final = workers[0] + workers: Final = rig.proxy.worker_pids() + assert len(workers) >= 2, f"expected multiple uvicorn workers, got {sorted(workers)!r}" + victim: Final = min(workers) with ThreadPoolExecutor(max_workers=20) as pool: futures: Final = tuple(pool.submit(call, index) for index in range(20)) while sum(f.done() for f in futures) < 5: wait(futures, timeout=0.05) - victim.kill() + replacement: Final = rig.proxy.kill_worker(victim) outcomes: Final = tuple(f.result() for f in futures) - assert not victim.is_running() or victim.status() == psutil.STATUS_ZOMBIE + assert replacement != victim succeeded: Final = tuple(identifier for status, identifier in outcomes if status == 200) assert succeeded, outcomes post_kill: Final = tuple(call(20 + index) for index in range(5)) diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index c5e3019cfda..e658f24b041 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -834,26 +834,22 @@ def test_langfuse_mapper_skips_empty_identity_values(): assert attrs["langfuse.trace.metadata.user_api_key_end_user_id"] == "end-1" -def test_langfuse_mapper_redacts_user_api_key_fields(): +def test_langfuse_mapper_redacts_user_api_key_fields(monkeypatch: pytest.MonkeyPatch) -> None: import litellm - saved: Final = litellm.redact_user_api_key_info - try: - litellm.redact_user_api_key_info = True - payload: Final[dict[str, object]] = { - "call_type": "acompletion", - "custom_llm_provider": "openai", - "model": "gpt-4o", - "metadata": { - "user_api_key_alias": "prod-key", - "user_api_key_team_id": "team-9", - "spend_logs_metadata": {"ticket": "LIT-8283"}, - }, - "response": {}, - } - attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) - finally: - litellm.redact_user_api_key_info = saved + monkeypatch.setattr(litellm, "redact_user_api_key_info", True) + payload: Final[dict[str, object]] = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "metadata": { + "user_api_key_alias": "prod-key", + "user_api_key_team_id": "team-9", + "spend_logs_metadata": {"ticket": "LIT-8283"}, + }, + "response": {}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) exported: Final = json.loads(attrs["langfuse.observation.metadata"]) assert not [key for key in exported if key.startswith("user_api_key")] diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 183f21bb806..ab0e7669fe5 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -500,7 +500,7 @@ class TestLangfuseOtelIntegration: "requester_ip_address": "10.0.0.1", "requester_metadata": {"ticket": "LIT-8283"}, } - kwargs: Final = { + kwargs: Final[dict[str, object]] = { "litellm_params": {"metadata": {}}, "standard_logging_object": {"metadata": request_metadata}, } @@ -519,9 +519,8 @@ class TestLangfuseOtelIntegration: "langfuse.trace.metadata.user_api_key_team_alias": "team-a", } - def test_request_metadata_redaction_matches_vanilla_langfuse(self) -> None: - previous_flag: Final = litellm.redact_user_api_key_info - litellm.redact_user_api_key_info = True + def test_request_metadata_redaction_matches_vanilla_langfuse(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "redact_user_api_key_info", True) request_metadata: Final = { "user_api_key_hash": "hash123", "user_api_key_alias": "prod-key", @@ -533,25 +532,22 @@ class TestLangfuseOtelIntegration: "requester_ip_address": "10.0.0.1", "requester_metadata": {"ticket": "LIT-8283"}, } - kwargs: Final = { + kwargs: Final[dict[str, object]] = { "litellm_params": {"metadata": {}}, "standard_logging_object": {"metadata": request_metadata}, } - try: - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - assert json.loads(actual["langfuse.observation.metadata"]) == { - "spend_logs_metadata": {"env": "prod"}, - "requester_ip_address": "10.0.0.1", - "requester_metadata": {"ticket": "LIT-8283"}, - } - assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] - finally: - litellm.redact_user_api_key_info = previous_flag + assert json.loads(actual["langfuse.observation.metadata"]) == { + "spend_logs_metadata": {"env": "prod"}, + "requester_ip_address": "10.0.0.1", + "requester_metadata": {"ticket": "LIT-8283"}, + } + assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] def test_request_metadata_drops_null_fields_and_skips_empty_trace_identities(self) -> None: request_metadata: Final = { @@ -564,7 +560,7 @@ class TestLangfuseOtelIntegration: "team_alias": None, "spend_logs_metadata": {"env": "prod"}, } - kwargs: Final = { + kwargs: Final[dict[str, object]] = { "litellm_params": {"metadata": {}}, "standard_logging_object": {"metadata": request_metadata}, } From acb7e42bdb2bbdb1eeb9f7f617cce38b135ad8d7 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 16 Oct 2026 15:25:18 +0000 Subject: [PATCH 20/28] fix(langfuse otel): drop prompt variables, MCP tool I/O and vector store queries from exported request metadata --- litellm/integrations/otel/model/metadata.py | 37 +++++++++++++- .../otel/test_otel_v2_vendor_mappers.py | 50 ++++++++++++++++++ tests/unit/integrations/test_langfuse_otel.py | 51 +++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 5047cbfe979..6ade4bdb539 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -60,6 +60,9 @@ if TYPE_CHECKING: _INTERNAL_CALL_ORIGINS: Final[frozenset[str]] = frozenset(get_args(InternalCallOrigin)) _AUTH_ATTRIBUTE: Final = TypeAdapter(object) _REQUEST_METADATA_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_PROMPT_MANAGEMENT_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"prompt_variables"}) +_MCP_TOOL_CALL_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"arguments", "result"}) +_VECTOR_STORE_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"query", "vector_store_search_response"}) REQUESTER_METADATA_KEY: Final = "requester_metadata" REQUESTER_METADATA_PATH: Final = f"{REQUESTER_METADATA_KEY}." @@ -484,7 +487,39 @@ def exported_request_metadata(payload: Mapping[str, object]) -> Mapping[str, obj ) redacted: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(redact_metadata(dict(metadata))) exported: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(log_requester_metadata(redacted)) - return MappingProxyType({key: value for key, value in exported.items() if value is not None}) + null_filtered: Final = MappingProxyType({key: value for key, value in exported.items() if value is not None}) + return _without_request_metadata_content(null_filtered) + + +def _without_mapping_content_fields(value: object, content_fields: frozenset[str]) -> object: + if not isinstance(value, dict): + return value + mapping: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(value) + return {key: item for key, item in mapping.items() if key not in content_fields} + + +def _without_vector_store_content_fields(value: object) -> object: + if not isinstance(value, list): + return value + return tuple(_without_mapping_content_fields(item, _VECTOR_STORE_CONTENT_FIELDS) for item in value) + + +def _request_metadata_value_without_content(key: str, value: object) -> object: + match key: + case "prompt_management_metadata": + return _without_mapping_content_fields(value, _PROMPT_MANAGEMENT_CONTENT_FIELDS) + case "mcp_tool_call_metadata": + return _without_mapping_content_fields(value, _MCP_TOOL_CALL_CONTENT_FIELDS) + case "vector_store_request_metadata": + return _without_vector_store_content_fields(value) + case _: + return value + + +def _without_request_metadata_content(metadata: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + {key: _request_metadata_value_without_content(key, value) for key, value in metadata.items()} + ) def _team_metadata_dict(value: object) -> Mapping[str, object] | None: diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index e658f24b041..bdc0b47586f 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -815,6 +815,56 @@ def test_langfuse_mapper_emits_request_metadata_and_identity(): assert attrs["langfuse.observation.metadata.provider"] == "openai" +def test_langfuse_mapper_omits_prompt_mcp_and_vector_store_content(): + payload: Final[dict[str, object]] = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "metadata": { + "prompt_management_metadata": { + "prompt_id": "prompt-1", + "prompt_variables": {"input": "prompt-variables-secret"}, + "prompt_integration": "prompt-service", + }, + "mcp_tool_call_metadata": { + "name": "lookup_weather", + "mcp_server_name": "weather-server", + "arguments": {"location": "mcp-arguments-secret"}, + "result": {"forecast": "mcp-result-secret"}, + }, + "vector_store_request_metadata": [ + { + "vector_store_id": "vs-1", + "custom_llm_provider": "openai", + "query": "vector-query-secret", + "vector_store_search_response": {"text": "vector-response-secret"}, + "start_time": 1.0, + "end_time": 2.0, + } + ], + }, + "response": {}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + observation_metadata: Final = attrs["langfuse.observation.metadata"] + serialized: Final = json.dumps(observation_metadata) + parsed: Final = json.loads(observation_metadata) + + assert "prompt-variables-secret" not in serialized + assert "mcp-arguments-secret" not in serialized + assert "mcp-result-secret" not in serialized + assert "vector-query-secret" not in serialized + assert "vector-response-secret" not in serialized + assert parsed["prompt_management_metadata"]["prompt_id"] == "prompt-1" + assert parsed["prompt_management_metadata"]["prompt_integration"] == "prompt-service" + assert parsed["mcp_tool_call_metadata"]["name"] == "lookup_weather" + assert parsed["mcp_tool_call_metadata"]["mcp_server_name"] == "weather-server" + assert parsed["vector_store_request_metadata"][0]["vector_store_id"] == "vs-1" + assert parsed["vector_store_request_metadata"][0]["custom_llm_provider"] == "openai" + assert parsed["vector_store_request_metadata"][0]["start_time"] == 1.0 + assert parsed["vector_store_request_metadata"][0]["end_time"] == 2.0 + + def test_langfuse_mapper_skips_empty_identity_values(): payload: Final[dict[str, object]] = { "call_type": "acompletion", diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index ab0e7669fe5..508d2cf9160 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -549,6 +549,57 @@ class TestLangfuseOtelIntegration: } assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] + def test_request_metadata_omits_prompt_mcp_and_vector_store_content(self) -> None: + request_metadata: Final = { + "prompt_management_metadata": { + "prompt_id": "prompt-1", + "prompt_variables": {"input": "prompt-variables-secret"}, + "prompt_integration": "prompt-service", + }, + "mcp_tool_call_metadata": { + "name": "lookup_weather", + "mcp_server_name": "weather-server", + "arguments": {"location": "mcp-arguments-secret"}, + "result": {"forecast": "mcp-result-secret"}, + }, + "vector_store_request_metadata": [ + { + "vector_store_id": "vs-1", + "custom_llm_provider": "openai", + "query": "vector-query-secret", + "vector_store_search_response": {"text": "vector-response-secret"}, + "start_time": 1.0, + "end_time": 2.0, + } + ], + } + kwargs: Final[dict[str, object]] = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": {"metadata": request_metadata}, + } + + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + observation_metadata: Final = actual["langfuse.observation.metadata"] + serialized: Final = json.dumps(observation_metadata) + parsed: Final = json.loads(observation_metadata) + + assert "prompt-variables-secret" not in serialized + assert "mcp-arguments-secret" not in serialized + assert "mcp-result-secret" not in serialized + assert "vector-query-secret" not in serialized + assert "vector-response-secret" not in serialized + assert parsed["prompt_management_metadata"]["prompt_id"] == "prompt-1" + assert parsed["prompt_management_metadata"]["prompt_integration"] == "prompt-service" + assert parsed["mcp_tool_call_metadata"]["name"] == "lookup_weather" + assert parsed["mcp_tool_call_metadata"]["mcp_server_name"] == "weather-server" + assert parsed["vector_store_request_metadata"][0]["vector_store_id"] == "vs-1" + assert parsed["vector_store_request_metadata"][0]["custom_llm_provider"] == "openai" + assert parsed["vector_store_request_metadata"][0]["start_time"] == 1.0 + assert parsed["vector_store_request_metadata"][0]["end_time"] == 2.0 + def test_request_metadata_drops_null_fields_and_skips_empty_trace_identities(self) -> None: request_metadata: Final = { "user_api_key_alias": "prod-key", From 316bdd8988bd8ece85155cdaa180a47d65d775ec Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 17 Oct 2026 05:42:47 -0700 Subject: [PATCH 21/28] fix(langfuse_otel): keep span export alive when nested request metadata has non-string keys The content strip validated prompt_management_metadata, mcp_tool_call_metadata and vector_store_request_metadata items as str-keyed mappings, so an SDK caller passing one with a non-string key raised inside exported_request_metadata: OTEL v2 dropped the LLM span for every vendor and v1 langfuse_otel lost the request metadata and every attribute after it. Validate those nested mappings with object keys instead. Also hoists the langfuse_otel imports. --- .../integrations/langfuse/langfuse_otel.py | 12 ++++----- litellm/integrations/otel/model/metadata.py | 3 ++- .../otel/test_otel_v2_vendor_mappers.py | 21 ++++++++++++++++ tests/unit/integrations/test_langfuse_otel.py | 25 +++++++++++++++++++ 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index a663356ad7b..aa83b76335a 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -13,6 +13,9 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import ( LangfuseLLMObsOTELAttributes, ) from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.integrations.otel.mappers.langfuse import TRACE_IDENTITY_FIELDS +from litellm.integrations.otel.model.metadata import exported_request_metadata +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, @@ -135,11 +138,6 @@ class LangfuseOtelLogger(OpenTelemetry): span: Span, kwargs: dict[str, object], # mutable-ok: required callback signature accepts kwargs as a dict ) -> None: - from litellm.integrations.arize._utils import safe_set_attribute - from litellm.integrations.otel.mappers.langfuse import TRACE_IDENTITY_FIELDS - from litellm.integrations.otel.model.metadata import exported_request_metadata - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - raw_standard_logging_object: Final = kwargs.get("standard_logging_object") if not isinstance(raw_standard_logging_object, dict): return @@ -148,7 +146,7 @@ class LangfuseOtelLogger(OpenTelemetry): return observation_metadata: Final = exported_request_metadata(standard_logging_object) if observation_metadata: - safe_set_attribute( + _utils.safe_set_attribute( span, LangfuseSpanAttributes.OBSERVATION_METADATA.value, safe_dumps(dict(observation_metadata)), @@ -156,7 +154,7 @@ class LangfuseOtelLogger(OpenTelemetry): trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value for field in TRACE_IDENTITY_FIELDS: if value := observation_metadata.get(field): - safe_set_attribute(span, f"{trace_prefix}.{field}", value) + _utils.safe_set_attribute(span, f"{trace_prefix}.{field}", value) @staticmethod def _set_observation_output(span: Span, response_obj): diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 6ade4bdb539..f3185387d68 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -60,6 +60,7 @@ if TYPE_CHECKING: _INTERNAL_CALL_ORIGINS: Final[frozenset[str]] = frozenset(get_args(InternalCallOrigin)) _AUTH_ATTRIBUTE: Final = TypeAdapter(object) _REQUEST_METADATA_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_NESTED_METADATA_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) _PROMPT_MANAGEMENT_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"prompt_variables"}) _MCP_TOOL_CALL_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"arguments", "result"}) _VECTOR_STORE_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"query", "vector_store_search_response"}) @@ -494,7 +495,7 @@ def exported_request_metadata(payload: Mapping[str, object]) -> Mapping[str, obj def _without_mapping_content_fields(value: object, content_fields: frozenset[str]) -> object: if not isinstance(value, dict): return value - mapping: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(value) + mapping: Final = _NESTED_METADATA_MAPPING_ADAPTER.validate_python(value) return {key: item for key, item in mapping.items() if key not in content_fields} diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index bdc0b47586f..a9351f6dfbf 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -919,3 +919,24 @@ def test_langfuse_mapper_omits_observation_metadata_without_payload_metadata(): attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) assert json.loads(attrs["langfuse.observation.metadata"]) == {"requester_metadata": {}} + + +@pytest.mark.parametrize( + "unexpected_shape", + [ + {"prompt_management_metadata": {1: "one", "prompt_id": "prompt-1"}}, + {"mcp_tool_call_metadata": {1: "one", "name": "lookup_weather"}}, + {"vector_store_request_metadata": [{1: "one", "vector_store_id": "vs-1"}]}, + ], +) +def test_span_data_builds_from_request_metadata_of_unexpected_shape(unexpected_shape: dict[str, object]) -> None: + payload: Final[dict[str, object]] = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "metadata": {**unexpected_shape, "spend_logs_metadata": {"ticket": "LIT-8283"}}, + "response": {}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + + assert json.loads(attrs["langfuse.observation.metadata"])["spend_logs_metadata"] == {"ticket": "LIT-8283"} diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 508d2cf9160..04d37738cdb 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -644,6 +644,31 @@ class TestLangfuseOtelIntegration: assert [key for key in actual if key.startswith("langfuse.trace.metadata")] == ["langfuse.trace.metadata"] assert "langfuse.observation.metadata" not in actual + @pytest.mark.parametrize( + "unexpected_shape", + [ + {"prompt_management_metadata": {1: "one", "prompt_id": "prompt-1"}}, + {"mcp_tool_call_metadata": {1: "one", "name": "lookup_weather"}}, + {"vector_store_request_metadata": [{1: "one", "vector_store_id": "vs-1"}]}, + ], + ) + def test_request_metadata_of_unexpected_shape_keeps_the_observation_output( + self, unexpected_shape: dict[str, object] + ) -> None: + kwargs: Final[dict[str, object]] = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": {"metadata": {**unexpected_shape, "spend_logs_metadata": {"env": "prod"}}}, + } + response_obj: Final = {"choices": [{"message": {"role": "assistant", "content": "hello"}}]} + + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, response_obj) + + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + + assert json.loads(actual["langfuse.observation.metadata"])["spend_logs_metadata"] == {"env": "prod"} + assert json.loads(actual["langfuse.observation.output"]) == {"role": "assistant", "content": "hello"} + class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target From c2646135781b4b6b4cbe50a221dce472413550ce Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 17 Oct 2026 14:44:52 -0700 Subject: [PATCH 22/28] fix(langfuse_otel): send the request metadata blob to Langfuse destinations only - A requester_metadata that is not a mapping no longer raises. The v2 LLM span (every vendor) and the v1 Langfuse attributes survive it again. - Prompt-quoting routing_decision fields stay out of langfuse.observation.metadata. - An exporter or key/team destination for any other backend drops langfuse.observation.metadata and langfuse.trace.metadata.user_api_key_*, as on main. This covers a collector folded into the Langfuse logger. - The v2 Langfuse key-set pin in the arize family test gains the two new keys. --- litellm/integrations/otel/mappers/langfuse.py | 4 + litellm/integrations/otel/model/metadata.py | 14 +- .../integrations/otel/plumbing/providers.py | 59 ++++- ...test_arize_otel_v2_openinference_family.py | 2 + .../test_langfuse_otel_metadata.py | 217 +++++++++++++++++- .../test_otel_team_capture_message_content.py | 21 ++ .../otel/test_otel_v2_destinations.py | 81 +++++++ .../otel/test_otel_v2_vendor_mappers.py | 33 +++ tests/unit/integrations/test_langfuse_otel.py | 30 +++ 9 files changed, 451 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 4f65111c4d8..bfd56f42efe 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -50,6 +50,10 @@ TRACE_IDENTITY_FIELDS: Final = ( "user_api_key_team_id", "user_api_key_team_alias", ) +# The request metadata attributes only a Langfuse destination receives. +LANGFUSE_REQUEST_METADATA_ATTRIBUTES: Final[frozenset[str]] = frozenset( + {LANGFUSE_OBSERVATION_METADATA, *(f"{LANGFUSE_TRACE_METADATA_PREFIX}{name}" for name in TRACE_IDENTITY_FIELDS)} +) def _identity_field(name: str) -> Callable[[LLMCallSpanData], AttrValue | None]: diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index f3185387d68..7cb8995c0e8 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -52,7 +52,7 @@ from litellm.constants import ( from litellm.integrations.otel.model.semconv import resolve_operation from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds -from litellm.types.utils import InternalCallOrigin +from litellm.types.utils import PROMPT_QUOTING_ROUTING_DECISION_FIELDS, InternalCallOrigin if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload @@ -472,7 +472,8 @@ def _model_info_id(model_info: object) -> str | None: def exported_request_metadata(payload: Mapping[str, object]) -> Mapping[str, object]: """The request metadata as the logging callbacks export it: ``user_api_key_*`` dropped when ``litellm.redact_user_api_key_info`` is on, header keys nested - under ``requester_metadata``, ``None`` values dropped.""" + under a mapping ``requester_metadata`` (any other value is exported as sent), + ``None`` values dropped.""" from litellm.integrations.langfuse.langfuse import log_requester_metadata from litellm.litellm_core_utils.redact_messages import ( # pyright: ignore[reportUnknownVariableType] # helper uses bare dict redact_user_api_key_info as _redact_user_api_key_info, @@ -487,7 +488,12 @@ def exported_request_metadata(payload: Mapping[str, object]) -> Mapping[str, obj _redact_user_api_key_info, ) redacted: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(redact_metadata(dict(metadata))) - exported: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(log_requester_metadata(redacted)) + requester_metadata: Final = redacted.get(REQUESTER_METADATA_KEY) + exported: Final = ( + _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(log_requester_metadata(redacted)) + if requester_metadata is None or isinstance(requester_metadata, Mapping) + else redacted + ) null_filtered: Final = MappingProxyType({key: value for key, value in exported.items() if value is not None}) return _without_request_metadata_content(null_filtered) @@ -513,6 +519,8 @@ def _request_metadata_value_without_content(key: str, value: object) -> object: return _without_mapping_content_fields(value, _MCP_TOOL_CALL_CONTENT_FIELDS) case "vector_store_request_metadata": return _without_vector_store_content_fields(value) + case "routing_decision": + return _without_mapping_content_fields(value, PROMPT_QUOTING_ROUTING_DECISION_FIELDS) case _: return value diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 898eab5715b..0f298d572b0 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -43,6 +43,7 @@ from litellm._version import version as litellm_version from litellm.integrations.otel.mappers.langfuse import ( LANGFUSE_OBSERVATION_INPUT, LANGFUSE_OBSERVATION_OUTPUT, + LANGFUSE_REQUEST_METADATA_ATTRIBUTES, LANGFUSE_TRACE_NAME, ) from litellm.integrations.otel.model.config import ( @@ -459,6 +460,17 @@ def without_message_content(span: ReadableSpan) -> ReadableSpan: return _SpanView(span, span.resource, kept, span.events, span.status, parent=span.parent) +def without_langfuse_request_metadata(span: ReadableSpan) -> ReadableSpan: + """``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination.""" + attributes: Final = span.attributes or _NO_ATTRIBUTES + if not any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes): + return span + kept: Final = MappingProxyType( + {key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES} + ) + return _SpanView(span, span.resource, kept, span.events, span.status, parent=span.parent) + + def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: return any(key in attributes for key in _DB_SYSTEM_KEYS) @@ -614,8 +626,9 @@ def _for_destination( captures on the server span, whose request side holds the caller's bearer token, and the query string of the request URL, which can hold the same key. A destination whose effective setting disables span content loses the prompt and - response content too. The span itself stays, so the tenant still gets the whole - trace tree. + response content too, and a destination for any backend but Langfuse loses the + request metadata the Langfuse mapper writes. The span itself stays, so the tenant + still gets the whole trace tree. """ extra: Final = destination.resource_attributes defaults: Final = destination.resource_defaults @@ -624,11 +637,14 @@ def _for_destination( owned: Final = _is_tenant_owned_span(attributes) unreachable: Final = _guardrail_unreachable(attributes) redacted: Final = not destination.captures_content(default_capture) + langfuse: Final = destination.callback_name == ExporterOwner.LANGFUSE_OTEL kept: Final = MappingProxyType( { key: _without_query(key, value) for key, value in attributes.items() - if _tenant_visible(key, database, owned, unreachable) and not (redacted and _is_message_content(key)) + if _tenant_visible(key, database, owned, unreachable) + and not (redacted and _is_message_content(key)) + and (langfuse or key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES) } ) recorded: Final = span.events @@ -1106,6 +1122,25 @@ class _OverriddenBackendFilter(SpanProcessor): return self._inner.force_flush(timeout_millis) +class _LangfuseRequestMetadataFilter(SpanProcessor): + """Strip the Langfuse request metadata from every span before ``inner`` exports it.""" + + def __init__(self, inner: SpanProcessor) -> None: + self._inner: Final = inner + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + self._inner.on_start(span, parent_context) + + def on_end(self, span: ReadableSpan) -> None: + self._inner.on_end(without_langfuse_request_metadata(span)) + + def shutdown(self) -> None: + self._inner.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return self._inner.force_flush(timeout_millis) + + class _MessageContentFilter(SpanProcessor): """Strip prompt and response content from every span before ``inner`` exports it.""" @@ -1377,13 +1412,17 @@ def build_resource(config: OpenTelemetryV2Config) -> Resource: def _spec_processor( - spec: ExporterSpec, use_simple_processor: bool | None, strip_message_content: bool + spec: ExporterSpec, + use_simple_processor: bool | None, + strip_message_content: bool, + strip_langfuse_request_metadata: bool, ) -> SpanProcessor: exporting: Final = _processor_for( _exporter_from_spec(spec), (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), ) - return _MessageContentFilter(exporting) if strip_message_content else exporting + scoped: Final = _LangfuseRequestMetadataFilter(exporting) if strip_langfuse_request_metadata else exporting + return _MessageContentFilter(scoped) if strip_message_content else scoped def build_tracer_provider( @@ -1411,6 +1450,8 @@ def build_tracer_provider( ``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel`` alone; a collector or any other backend in the same config keeps the full tree. + In a config with a ``langfuse_otel`` exporter, every other exporter goes without + the request metadata the Langfuse mapper writes. ``strip_message_content`` removes prompt and response content before every exporter, for a routed provider whose spans can carry content another destination asked for. @@ -1426,10 +1467,16 @@ def build_tracer_provider( # ``config._normalize`` guarantees at least one spec (it folds the top-level # ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty). + langfuse_owned: Final = any(spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters) for spec in config.exporters: if spec.requires_headers and not spec.headers: continue - processor = _spec_processor(spec, use_simple_processor, strip_message_content) + processor = _spec_processor( + spec, + use_simple_processor, + strip_message_content, + strip_langfuse_request_metadata=langfuse_owned and spec.owner is not ExporterOwner.LANGFUSE_OTEL, + ) owner = spec.owner.value if tenant_overrides and spec.owner is not None else None scope = _operator_scope(config, spec) sink = _sink_key(spec.endpoint, parse_headers(spec.headers)) if _exports_to_the_wire(spec) else None diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_family.py b/tests/integration/observability/test_arize_otel_v2_openinference_family.py index 242b7bfe8c1..020ef1568df 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_family.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_family.py @@ -88,11 +88,13 @@ _LANGFUSE_B3_KEYS: Final = frozenset( "langfuse.observation.cost_details", "langfuse.observation.id", "langfuse.observation.input", + "langfuse.observation.metadata", "langfuse.observation.metadata.provider", "langfuse.observation.model.name", "langfuse.observation.output", "langfuse.observation.type", "langfuse.observation.usage_details", + "langfuse.trace.metadata.user_api_key_user_id", "litellm.api_key.hash", "litellm.call_id", "litellm.call_type", diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 68cbf7e5e89..23344cd800c 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -3,7 +3,7 @@ import binascii import json import threading import uuid -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor, wait from contextlib import contextmanager from dataclasses import dataclass @@ -11,6 +11,7 @@ from pathlib import Path from typing import Final import httpx +import pytest import yaml from anthropic import Anthropic, AsyncAnthropic from integration._support.client import Gateway, Scenario, eventually @@ -206,6 +207,7 @@ def _langfuse_rig( *, sink_reply: Callable[[Request], Reply] | None = None, overrides: Mapping[str, str] | None = None, + model_list: Callable[[str], Sequence[Mapping[str, object]]] | None = None, ) -> Iterator[_Rig]: def upstream(request: Request) -> Reply: if request.method == "GET": @@ -240,6 +242,7 @@ def _langfuse_rig( raw_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) config: Final = { **raw_config, + "model_list": [*raw_config.get("model_list", []), *(model_list(provider.url) if model_list else ())], "litellm_settings": {**raw_config["litellm_settings"], "callbacks": ["langfuse_otel"], **settings}, } path: Final = tmp_path / "langfuse_otel.yaml" @@ -292,6 +295,16 @@ def _watched_spans(collector: Wire) -> Callable[[], tuple[dict[str, object], ... return snapshot +def _v2_llm_span(collector: Wire, response_id: str) -> dict[str, object]: + def matches(span: Mapping[str, object]) -> bool: + return _response_id(span.get("gen_ai.response.id")) == _response_id(response_id) + + observed: Final = eventually( + _watched_spans(collector), lambda spans: sum(matches(span) for span in spans) == 1, seconds=25 + ) + return next(span for span in observed if matches(span)) + + def _observation_span(collector: Wire, response_id: str) -> dict[str, object]: def spans() -> tuple[dict[str, object], ...]: return tuple( @@ -1122,3 +1135,205 @@ def test_langfuse_otel_v2_mapper_teamless_key_emits_no_team_identity(gateway: Ga for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) if attribute in attrs } == {"langfuse.trace.metadata.user_api_key_alias": identity["key_alias"]}, attrs + + +def _scalar_requester_metadata_request(route: str, model: str, marker: str) -> dict[str, object]: + caller: Final = {"litellm_metadata": {"requester_metadata": 1}} + if route == "/v1/messages": + return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}], **caller} + return {"model": model, "input": marker, **caller} + + +@pytest.mark.parametrize( + ("callback", "route"), + [ + ("otel", "/v1/messages"), + ("otel", "/v1/responses"), + ("langfuse_otel", "/v1/messages"), + ("langfuse_otel", "/v1/responses"), + ], +) +def test_otel_v2_scalar_requester_metadata_keeps_the_llm_span( + callback: str, route: str, gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-v2-scalar-" + uuid.uuid4().hex + with ( + _langfuse_rig( + gateway, + tmp_path, + {"callbacks": [callback]}, + marker, + overrides={ + "LITELLM_OTEL_V2": "1", + **({"OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": "__collector__"} if callback == "otel" else {}), + }, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + model: Final = ( + scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) + if route == "/v1/messages" + else identity["model"] + ) + response: Final = rig.candidate.request( + "POST", route, _scalar_requester_metadata_request(route, str(model), marker), key=key + ) + assert response.status_code == 200, response.text + + attrs: Final = _v2_llm_span(rig.collector, response.json()["id"]) + assert attrs.get("gen_ai.usage.input_tokens") == 3, attrs + if callback == "langfuse_otel": + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + assert observation["requester_metadata"] == 1, observation + assert observation["user_api_key_alias"] == identity["key_alias"], observation + + +def test_langfuse_otel_scalar_requester_metadata_keeps_the_observation_input(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-scalar-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) + response: Final = rig.candidate.request( + "POST", "/v1/messages", _scalar_requester_metadata_request("/v1/messages", model, marker), key=key + ) + assert response.status_code == 200, response.text + + attrs: Final = _observation_span(rig.collector, response.json()["id"]) + assert marker in str(attrs.get("langfuse.observation.input")), sorted(attrs) + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + assert observation["requester_metadata"] == 1, observation + assert observation["user_api_key_alias"] == identity["key_alias"], observation + + +@pytest.mark.parametrize( + "callbacks", + [["langfuse_otel", "otel"], ["langfuse_otel"]], + ids=["langfuse-listed-before-otel", "langfuse-with-otel-env"], +) +def test_otel_v2_collector_beside_langfuse_gets_no_langfuse_request_metadata( + callbacks: list[str], gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-v2-generic-" + uuid.uuid4().hex + probe: Final = "lf-probe-" + uuid.uuid4().hex + with ( + wire_server(lambda _request: Reply()) as generic, + _langfuse_rig( + gateway, + tmp_path, + {"callbacks": callbacks}, + marker, + overrides={"LITELLM_OTEL_V2": "1", "OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": generic.url}, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "metadata": {"caller_note": probe}, + "cache": {"no-cache": True}, + }, + key=key, + headers={"x-lit8283-probe": probe}, + ) + assert response.status_code == 200, response.text + + langfuse: Final = _v2_llm_span(rig.collector, response.json()["id"]) + observation: Final = json.loads(str(langfuse["langfuse.observation.metadata"])) + assert observation["user_api_key_alias"] == identity["key_alias"], observation + assert langfuse["langfuse.trace.metadata.user_api_key_alias"] == identity["key_alias"], langfuse + + collector: Final = _v2_llm_span(generic, response.json()["id"]) + assert "langfuse.observation.metadata" not in collector, sorted(collector) + assert not [name for name in collector if name.startswith("langfuse.trace.metadata.user_api_key_")], sorted( + collector + ) + assert probe not in json.dumps(collector, default=str), collector + assert collector["langfuse.trace.metadata.team_id"] == identity["team_id"], collector + assert collector.get("gen_ai.usage.input_tokens") == 3, collector + + +_PROMPT_QUOTING_ROUTING_FIELDS: Final = ("signals", "matched_keyword", "escalation_keyword", "classifier_crux") + + +def _keyword_router(router: str, keyword: str) -> Callable[[str], Sequence[Mapping[str, object]]]: + def models(provider_url: str) -> Sequence[Mapping[str, object]]: + tier: Final = f"{router}-tier" + return ( + { + "model_name": tier, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "integration-provider-key", + "api_base": provider_url + "/v1", + }, + }, + { + "model_name": router, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "heuristic", + "keyword_tier_rules": [{"keywords": [keyword], "tier": "REASONING"}], + "tiers": {"SIMPLE": tier, "MEDIUM": tier, "COMPLEX": tier, "REASONING": tier}, + }, + }, + }, + ) + + return models + + +@pytest.mark.parametrize("otel_v2", [False, True], ids=["v1", "v2"]) +def test_langfuse_otel_metadata_omits_prompt_quoting_routing_fields( + otel_v2: bool, gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-route-" + uuid.uuid4().hex + router: Final = "lf-router-" + uuid.uuid4().hex[:8] + keyword: Final = "acquire-globex-" + uuid.uuid4().hex[:8] + with ( + _langfuse_rig( + gateway, + tmp_path, + {}, + marker, + overrides=( + {"LITELLM_OTEL_V2": "1", "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "no_content"} + if otel_v2 + else None + ), + model_list=_keyword_router(router, keyword), + ) as rig, + rig.candidate.scenario() as scenario, + ): + key: Final = scenario.key(key_alias="lf-route-" + uuid.uuid4().hex) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": router, + "messages": [{"role": "user", "content": f"{marker} please {keyword} now"}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + + attrs: Final = ( + _v2_llm_span(rig.collector, response.json()["id"]) + if otel_v2 + else _observation_span(rig.collector, response.json()["id"]) + ) + routing: Final = json.loads(str(attrs["langfuse.observation.metadata"]))["routing_decision"] + assert routing["tier"] == "REASONING", routing + assert routing["router_model_name"] == router, routing + assert not [field for field in _PROMPT_QUOTING_ROUTING_FIELDS if field in routing], routing + if otel_v2: + assert keyword not in json.dumps(attrs, default=str), attrs diff --git a/tests/integration/observability/test_otel_team_capture_message_content.py b/tests/integration/observability/test_otel_team_capture_message_content.py index 84da9a40486..c964dd3a772 100644 --- a/tests/integration/observability/test_otel_team_capture_message_content.py +++ b/tests/integration/observability/test_otel_team_capture_message_content.py @@ -465,6 +465,27 @@ def test_arize_destination_drops_openinference_content(rig: Rig) -> None: assert model.get("llm.model_name"), sorted(model) +def test_arize_destination_gets_no_langfuse_request_metadata(rig: Rig) -> None: + team, key = rig.team_key( + None, "arize-meta", callback="arize", variables={"arize_space_key": "space-meta", "arize_api_key": "arize-meta"} + ) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = _arrived(rig.sinks.operator, cursors.operator, sent, frozenset()) + assert any("langfuse.observation.metadata" in span["attributes"] for span in _model_spans(operator)), _names( + operator + ) + tenant: Final = _arrived(rig.sinks.arize, cursors.arize, sent, frozenset(), ("arize-space-id", "space-meta")) + models: Final = [span["attributes"] for span in _model_spans(tenant)] + assert any(attributes.get("langfuse.trace.metadata.team_id") == team for attributes in models), models + assert not [ + name + for attributes in models + for name in attributes + if name == "langfuse.observation.metadata" or name.startswith("langfuse.trace.metadata.user_api_key_") + ], models + + def test_weave_destination_drops_its_content(rig: Rig) -> None: _, key = rig.team_key( "no_content", diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index c7a057cca5c..7b79dc034df 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -5067,3 +5067,84 @@ class TestArizeProjectRouting: monkeypatch.setenv("ARIZE_PROJECT_NAME", "gateway-prod") assert self._forwarded_projects() == {"gateway-prod"} + + +_LANGFUSE_REQUEST_METADATA_SPAN: Final = MappingProxyType( + { + "langfuse.observation.metadata": '{"requester_metadata": {"headers": {"x-tenant": "acme"}}}', + "langfuse.trace.metadata.user_api_key_alias": "prod-key", + "langfuse.trace.metadata.user_api_key_user_id": "user-1", + "langfuse.trace.metadata.team_id": "team-9", + "gen_ai.request.model": "gpt-4o", + } +) +_WITHOUT_LANGFUSE_REQUEST_METADATA: Final = MappingProxyType( + {"langfuse.trace.metadata.team_id": "team-9", "gen_ai.request.model": "gpt-4o"} +) + + +class TestLangfuseRequestMetadataStaysWithLangfuse: + """The request metadata the Langfuse mapper writes reaches a Langfuse destination and no other.""" + + @staticmethod + def _emit(provider: TracerProvider, destinations: tuple[OtelDestination, ...] = ()) -> None: + def run() -> None: + set_request_destinations(destinations) + with get_tracer(provider, "litellm").start_as_current_span("chat gpt-4o") as span: + span.set_attributes(_LANGFUSE_REQUEST_METADATA_SPAN) + + in_fresh_context(run) + + @staticmethod + def _provider( + monkeypatch: pytest.MonkeyPatch, *owners: ExporterOwner | None + ) -> tuple[TracerProvider, dict[ExporterOwner | None, InMemorySpanExporter]]: + exporters: dict[ExporterOwner | None, InMemorySpanExporter] = {} # mutable-ok: filled as specs build + kind: Final = "lit8283_request_metadata" + monkeypatch.setitem( + otel_providers._EXPORTER_FACTORIES, + kind, + lambda spec: exporters.setdefault(spec.owner, InMemorySpanExporter()), + ) + config: Final = OpenTelemetryV2Config( + mapper_names=["genai", "langfuse"], + exporters=[ExporterSpec(kind=kind, owner=owner) for owner in owners], + ) + return build_tracer_provider(config, use_simple_processor=True, tenant_overrides=True), exporters + + def test_a_collector_beside_the_langfuse_exporter_gets_no_langfuse_request_metadata( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + provider, exporters = self._provider(monkeypatch, None, ExporterOwner.LANGFUSE_OTEL) + + self._emit(provider) + + assert dict(by_name(exporters[ExporterOwner.LANGFUSE_OTEL])["chat gpt-4o"].attributes) == dict( + _LANGFUSE_REQUEST_METADATA_SPAN + ) + assert dict(by_name(exporters[None])["chat gpt-4o"].attributes) == dict(_WITHOUT_LANGFUSE_REQUEST_METADATA) + + def test_a_collector_with_no_langfuse_exporter_keeps_what_its_mappers_write( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + provider, exporters = self._provider(monkeypatch, None) + + self._emit(provider) + + assert dict(by_name(exporters[None])["chat gpt-4o"].attributes) == dict(_LANGFUSE_REQUEST_METADATA_SPAN) + + def test_the_fan_out_forwards_langfuse_request_metadata_to_langfuse_destinations_only(self) -> None: + langfuse_exporter, arize_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda destination: SimpleSpanProcessor( + langfuse_exporter if destination.callback_name == "langfuse_otel" else arize_exporter + ) + ) + ) + + self._emit(provider, (LANGFUSE_DEST, INHERITING_DEST)) + + assert dict(by_name(langfuse_exporter)["chat gpt-4o"].attributes) == dict(_LANGFUSE_REQUEST_METADATA_SPAN) + assert dict(by_name(arize_exporter)["chat gpt-4o"].attributes) == dict(_WITHOUT_LANGFUSE_REQUEST_METADATA) diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index a9351f6dfbf..f33e9dc0f40 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -927,6 +927,9 @@ def test_langfuse_mapper_omits_observation_metadata_without_payload_metadata(): {"prompt_management_metadata": {1: "one", "prompt_id": "prompt-1"}}, {"mcp_tool_call_metadata": {1: "one", "name": "lookup_weather"}}, {"vector_store_request_metadata": [{1: "one", "vector_store_id": "vs-1"}]}, + {"requester_metadata": 1}, + {"requester_metadata": True}, + {"requester_metadata": 1.5}, ], ) def test_span_data_builds_from_request_metadata_of_unexpected_shape(unexpected_shape: dict[str, object]) -> None: @@ -940,3 +943,33 @@ def test_span_data_builds_from_request_metadata_of_unexpected_shape(unexpected_s attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) assert json.loads(attrs["langfuse.observation.metadata"])["spend_logs_metadata"] == {"ticket": "LIT-8283"} + + +def test_langfuse_mapper_omits_prompt_quoting_routing_fields(): + payload: Final[dict[str, object]] = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "metadata": { + "routing_decision": { + "router_model_name": "smart-router", + "router_type": "complexity", + "cause": "literal_keyword_match", + "tier": "REASONING", + "signals": ["keyword (routing-prompt-secret)"], + "matched_keyword": "routing-prompt-secret", + "escalation_keyword": "routing-prompt-secret", + "classifier_crux": "routing-prompt-secret", + } + }, + "response": {}, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + + assert "routing-prompt-secret" not in json.dumps(attrs) + assert json.loads(attrs["langfuse.observation.metadata"])["routing_decision"] == { + "router_model_name": "smart-router", + "router_type": "complexity", + "cause": "literal_keyword_match", + "tier": "REASONING", + } diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 04d37738cdb..19215fe3fc4 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -650,6 +650,9 @@ class TestLangfuseOtelIntegration: {"prompt_management_metadata": {1: "one", "prompt_id": "prompt-1"}}, {"mcp_tool_call_metadata": {1: "one", "name": "lookup_weather"}}, {"vector_store_request_metadata": [{1: "one", "vector_store_id": "vs-1"}]}, + {"requester_metadata": 1}, + {"requester_metadata": True}, + {"requester_metadata": 1.5}, ], ) def test_request_metadata_of_unexpected_shape_keeps_the_observation_output( @@ -669,6 +672,33 @@ class TestLangfuseOtelIntegration: assert json.loads(actual["langfuse.observation.metadata"])["spend_logs_metadata"] == {"env": "prod"} assert json.loads(actual["langfuse.observation.output"]) == {"role": "assistant", "content": "hello"} + def test_observation_metadata_omits_prompt_quoting_routing_fields(self) -> None: + kwargs: Final[dict[str, object]] = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": { + "metadata": { + "routing_decision": { + "router_type": "complexity", + "tier": "REASONING", + "signals": ["keyword (routing-prompt-secret)"], + "matched_keyword": "routing-prompt-secret", + "escalation_keyword": "routing-prompt-secret", + "classifier_crux": "routing-prompt-secret", + } + } + }, + } + + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + + assert json.loads(actual["langfuse.observation.metadata"])["routing_decision"] == { + "router_type": "complexity", + "tier": "REASONING", + } + class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target From c9410b2aeb714fd33013dd0d9eeaccf48bb5ffdc Mon Sep 17 00:00:00 2001 From: yucheng Date: Sun, 18 Oct 2026 02:01:45 -0700 Subject: [PATCH 23/28] fix(langfuse_otel): export only caller identity under the Langfuse metadata keys - The Langfuse metadata keys carried the whole request metadata. On OTel v2 that sent request headers, caller metadata, user email, user agent and auth metadata to a Langfuse destination that never received them on main; on v1 it doubled each span's metadata payload, so big caller metadata doubled the export size - Both paths now export only the key alias, user, end user, team, team alias and org. On v2 that is the team plus the identity among the operator's `baggage_metadata_keys`, which v2 spans already carry as `litellm.metadata.*`. Everything else stays in the bare `metadata` attribute, as on main - `output_messages` returns a list again, and the arize v2 b3 key pin is back to main's set --- .../integrations/langfuse/langfuse_otel.py | 20 +- litellm/integrations/otel/mappers/langfuse.py | 15 +- litellm/integrations/otel/mappers/utils.py | 4 +- litellm/integrations/otel/model/metadata.py | 83 ++----- litellm/integrations/otel/model/payloads.py | 14 +- ...test_arize_otel_v2_openinference_family.py | 2 - .../test_langfuse_otel_metadata.py | 143 ++++++++---- .../otel/test_otel_v2_vendor_mappers.py | 213 +++++++----------- tests/unit/integrations/test_langfuse_otel.py | 134 +++++------ 9 files changed, 291 insertions(+), 337 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index aa83b76335a..a8651bc2555 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -14,8 +14,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import ( ) from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.integrations.otel.mappers.langfuse import TRACE_IDENTITY_FIELDS -from litellm.integrations.otel.model.metadata import exported_request_metadata -from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.integrations.otel.model.metadata import IDENTITY_METADATA_FIELDS, exported_request_metadata from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, @@ -30,7 +29,7 @@ else: Span = Any -_STANDARD_LOGGING_OBJECT_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) LANGFUSE_CLOUD_EU_ENDPOINT: Final = "https://cloud.langfuse.com/api/public/otel" LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/otel" @@ -136,20 +135,23 @@ class LangfuseOtelLogger(OpenTelemetry): @staticmethod def _set_request_metadata_attributes( span: Span, - kwargs: dict[str, object], # mutable-ok: required callback signature accepts kwargs as a dict + kwargs: Mapping[object, object], ) -> None: raw_standard_logging_object: Final = kwargs.get("standard_logging_object") if not isinstance(raw_standard_logging_object, dict): return - standard_logging_object: Final = _STANDARD_LOGGING_OBJECT_ADAPTER.validate_python(raw_standard_logging_object) - if not isinstance(standard_logging_object.get("metadata"), dict): + raw_metadata: Final = _MAPPING_ADAPTER.validate_python(raw_standard_logging_object).get("metadata") + if not isinstance(raw_metadata, dict): return - observation_metadata: Final = exported_request_metadata(standard_logging_object) + metadata: Final = _MAPPING_ADAPTER.validate_python(raw_metadata) + observation_metadata: Final = exported_request_metadata( + {field: metadata.get(field) for field in IDENTITY_METADATA_FIELDS} + ) if observation_metadata: _utils.safe_set_attribute( span, LangfuseSpanAttributes.OBSERVATION_METADATA.value, - safe_dumps(dict(observation_metadata)), + json.dumps(dict(observation_metadata)), ) trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value for field in TRACE_IDENTITY_FIELDS: @@ -276,7 +278,7 @@ class LangfuseOtelLogger(OpenTelemetry): metadata: Final = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata) - LangfuseOtelLogger._set_request_metadata_attributes(span=span, kwargs=kwargs) + LangfuseOtelLogger._set_request_metadata_attributes(span=span, kwargs=_MAPPING_ADAPTER.validate_python(kwargs)) messages: Final = kwargs.get("messages") if messages: diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index bfd56f42efe..647b18cbcad 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -11,12 +11,10 @@ the JSON-serialized payloads. ``trace_attributes`` maps the caller's trace contr """ import json -from collections.abc import Callable, Mapping +from collections.abc import Callable from types import MappingProxyType from typing import Final -from pydantic import TypeAdapter - from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData from litellm.integrations.otel.mappers.utils import ( collect, @@ -31,8 +29,6 @@ from litellm.integrations.otel.model.payloads import ( LLMUsage, ) from litellm.integrations.otel.model.trace_controls import TraceControls -from litellm.integrations.otel.model.utils import as_str -from litellm.litellm_core_utils.safe_json_dumps import safe_dumps LANGFUSE_OBSERVATION_INPUT: Final = "langfuse.observation.input" LANGFUSE_OBSERVATION_OUTPUT: Final = "langfuse.observation.output" @@ -42,7 +38,6 @@ LANGFUSE_TRACE_SESSION_ID: Final = "session.id" LANGFUSE_TRACE_TAGS: Final = "langfuse.trace.tags" LANGFUSE_OBSERVATION_METADATA: Final = "langfuse.observation.metadata" LANGFUSE_TRACE_METADATA_PREFIX: Final = "langfuse.trace.metadata." -_REQUEST_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) TRACE_IDENTITY_FIELDS: Final = ( "user_api_key_alias", "user_api_key_user_id", @@ -57,7 +52,7 @@ LANGFUSE_REQUEST_METADATA_ATTRIBUTES: Final[frozenset[str]] = frozenset( def _identity_field(name: str) -> Callable[[LLMCallSpanData], AttrValue | None]: - return lambda d: as_str(d.request_metadata.get(name)) or None + return lambda d: d.request_metadata.get(name) class LangfuseMapper: @@ -94,11 +89,7 @@ class LangfuseMapper: # JSON-payload attributes: each builder returns the serialized blob or None. _BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { - LANGFUSE_OBSERVATION_METADATA: lambda d: ( - safe_dumps({**_REQUEST_METADATA_ADAPTER.validate_python(d.request_metadata)}) - if d.request_metadata - else None - ), + LANGFUSE_OBSERVATION_METADATA: lambda d: json_if(dict(d.request_metadata)), "langfuse.observation.model.parameters": lambda d: json_if( collect(LangfuseMapper._MODEL_PARAMS, d.request_params) ), diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index c7e99ae0535..80ef2dd553a 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -135,9 +135,9 @@ def message_content(message: object) -> str | None: return None -def output_messages(data: LLMCallSpanData) -> tuple[object, ...]: +def output_messages(data: LLMCallSpanData) -> list: """The ``message`` payload of each response choice.""" - return tuple(c.get("message") for c in data.choices_out if isinstance(c, dict)) + return [c.get("message") for c in data.choices_out if isinstance(c, dict)] def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]: diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 7cb8995c0e8..4a888f9366a 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -52,18 +52,24 @@ from litellm.constants import ( from litellm.integrations.otel.model.semconv import resolve_operation from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds -from litellm.types.utils import PROMPT_QUOTING_ROUTING_DECISION_FIELDS, InternalCallOrigin +from litellm.types.utils import InternalCallOrigin if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload _INTERNAL_CALL_ORIGINS: Final[frozenset[str]] = frozenset(get_args(InternalCallOrigin)) _AUTH_ATTRIBUTE: Final = TypeAdapter(object) -_REQUEST_METADATA_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) -_NESTED_METADATA_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) -_PROMPT_MANAGEMENT_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"prompt_variables"}) -_MCP_TOOL_CALL_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"arguments", "result"}) -_VECTOR_STORE_CONTENT_FIELDS: Final[frozenset[str]] = frozenset({"query", "vector_store_search_response"}) +_EXPORTED_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, str]]] = TypeAdapter(Mapping[str, str]) +# The caller identity the Langfuse metadata keys export; the rest of the request metadata stays in the bare +# ``metadata`` attribute. +IDENTITY_METADATA_FIELDS: Final = ( + "user_api_key_alias", + "user_api_key_user_id", + "user_api_key_end_user_id", + "user_api_key_team_id", + "user_api_key_team_alias", + "user_api_key_org_id", +) REQUESTER_METADATA_KEY: Final = "requester_metadata" REQUESTER_METADATA_PATH: Final = f"{REQUESTER_METADATA_KEY}." @@ -469,66 +475,21 @@ def _model_info_id(model_info: object) -> str | None: return None -def exported_request_metadata(payload: Mapping[str, object]) -> Mapping[str, object]: - """The request metadata as the logging callbacks export it: ``user_api_key_*`` - dropped when ``litellm.redact_user_api_key_info`` is on, header keys nested - under a mapping ``requester_metadata`` (any other value is exported as sent), - ``None`` values dropped.""" - from litellm.integrations.langfuse.langfuse import log_requester_metadata - from litellm.litellm_core_utils.redact_messages import ( # pyright: ignore[reportUnknownVariableType] # helper uses bare dict - redact_user_api_key_info as _redact_user_api_key_info, +def exported_request_metadata(metadata: Mapping[str, object]) -> Mapping[str, str]: + """The non-empty string ``IDENTITY_METADATA_FIELDS`` of ``metadata``, with ``user_api_key_*`` dropped when + ``litellm.redact_user_api_key_info`` is on, as the ``langfuse`` callback does.""" + from litellm.litellm_core_utils.redact_messages import ( + redact_user_api_key_info as _redact_user_api_key_info, # pyright: ignore[reportUnknownVariableType] # helper uses bare dict ) - raw_metadata: Final = payload.get("metadata") - metadata: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python( - raw_metadata if isinstance(raw_metadata, Mapping) else MappingProxyType({}) - ) redact_metadata: Final = cast( # cast-ok: adapter validates the helper's output below - Callable[[dict[str, object]], object], + Callable[[dict[str, str]], object], _redact_user_api_key_info, ) - redacted: Final = _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(redact_metadata(dict(metadata))) - requester_metadata: Final = redacted.get(REQUESTER_METADATA_KEY) - exported: Final = ( - _REQUEST_METADATA_MAPPING_ADAPTER.validate_python(log_requester_metadata(redacted)) - if requester_metadata is None or isinstance(requester_metadata, Mapping) - else redacted - ) - null_filtered: Final = MappingProxyType({key: value for key, value in exported.items() if value is not None}) - return _without_request_metadata_content(null_filtered) - - -def _without_mapping_content_fields(value: object, content_fields: frozenset[str]) -> object: - if not isinstance(value, dict): - return value - mapping: Final = _NESTED_METADATA_MAPPING_ADAPTER.validate_python(value) - return {key: item for key, item in mapping.items() if key not in content_fields} - - -def _without_vector_store_content_fields(value: object) -> object: - if not isinstance(value, list): - return value - return tuple(_without_mapping_content_fields(item, _VECTOR_STORE_CONTENT_FIELDS) for item in value) - - -def _request_metadata_value_without_content(key: str, value: object) -> object: - match key: - case "prompt_management_metadata": - return _without_mapping_content_fields(value, _PROMPT_MANAGEMENT_CONTENT_FIELDS) - case "mcp_tool_call_metadata": - return _without_mapping_content_fields(value, _MCP_TOOL_CALL_CONTENT_FIELDS) - case "vector_store_request_metadata": - return _without_vector_store_content_fields(value) - case "routing_decision": - return _without_mapping_content_fields(value, PROMPT_QUOTING_ROUTING_DECISION_FIELDS) - case _: - return value - - -def _without_request_metadata_content(metadata: Mapping[str, object]) -> Mapping[str, object]: - return MappingProxyType( - {key: _request_metadata_value_without_content(key, value) for key, value in metadata.items()} - ) + present: Final = { + field: value for field in IDENTITY_METADATA_FIELDS if isinstance(value := metadata.get(field), str) and value + } + return MappingProxyType(dict(_EXPORTED_METADATA_ADAPTER.validate_python(redact_metadata(present)))) def _team_metadata_dict(value: object) -> Mapping[str, object] | None: diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 4bae881c434..bd6f7ac8d8d 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -422,7 +422,8 @@ class LLMCallSpanData: response_cost: float | None server: ServerInfo | None identity: RequestIdentity - request_metadata: Mapping[str, object] = field(default_factory=dict) + # The caller identity among the promoted metadata plus the team (see ``exported_request_metadata``). + request_metadata: Mapping[str, str] = field(default_factory=lambda: _EMPTY_METADATA) is_streaming: bool | None = None cost: LLMCost = field(default_factory=LLMCost) tools: tuple[ToolDefinition, ...] = () @@ -468,6 +469,7 @@ class LLMCallSpanData: context: Final = RequestContext.from_standard_logging_payload(payload) metadata: Final = payload.get("metadata") decision: Final = metadata["routing_decision"] if metadata and "routing_decision" in metadata else None + promoted_metadata: Final = allowlisted_metadata(context.identity.metadata, metadata_keys) # Normalize ``response`` to a dict once so the content/id reads below are a # plain ``.get`` — no repeated ``isinstance`` guards. raw_response: Final = payload.get("response") @@ -498,8 +500,14 @@ class LLMCallSpanData: cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))), server=ServerInfo.from_api_base(context.api_base), identity=context.identity, - request_metadata=exported_request_metadata(payload), - promoted_metadata=allowlisted_metadata(context.identity.metadata, metadata_keys), + request_metadata=exported_request_metadata( + { + "user_api_key_team_id": context.identity.team_id, + "user_api_key_team_alias": context.identity.team_alias, + **promoted_metadata, + } + ), + promoted_metadata=promoted_metadata, is_streaming=as_bool(payload.get("stream")), tools=_extract_tools(params), messages_in=_dicts(payload.get("messages")) if capture_content else (), diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_family.py b/tests/integration/observability/test_arize_otel_v2_openinference_family.py index 020ef1568df..242b7bfe8c1 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_family.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_family.py @@ -88,13 +88,11 @@ _LANGFUSE_B3_KEYS: Final = frozenset( "langfuse.observation.cost_details", "langfuse.observation.id", "langfuse.observation.input", - "langfuse.observation.metadata", "langfuse.observation.metadata.provider", "langfuse.observation.model.name", "langfuse.observation.output", "langfuse.observation.type", "langfuse.observation.usage_details", - "langfuse.trace.metadata.user_api_key_user_id", "litellm.api_key.hash", "litellm.call_id", "litellm.call_type", diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 23344cd800c..be867d07fe6 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -27,6 +27,8 @@ _IDENTITIES: Final = ( "user_api_key_team_id", "user_api_key_team_alias", ) +# The only request metadata the Langfuse metadata keys carry; everything else stays in the bare ``metadata``. +_EXPORTED_FIELDS: Final = frozenset({*_IDENTITIES, "user_api_key_org_id"}) @dataclass(frozen=True, slots=True) @@ -357,19 +359,13 @@ def _scenario_identity( } -def _assert_identity( - attrs: Mapping[str, object], - expected: Mapping[str, object], - *, - spend_logs_metadata: Mapping[str, object] | None = None, -) -> None: +def _assert_identity(attrs: Mapping[str, object], expected: Mapping[str, object]) -> None: assert "metadata" in attrs, sorted(attrs) assert "langfuse.observation.metadata" in attrs, sorted(attrs) observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) assert {key: observation.get(key) for key in expected} == dict(expected), observation - assert not [key for key, value in observation.items() if value is None], observation - if spend_logs_metadata is not None: - assert observation["spend_logs_metadata"] == dict(spend_logs_metadata), observation + assert set(observation) <= _EXPORTED_FIELDS, observation + assert all(isinstance(value, str) and value for value in observation.values()), observation flattened: Final = { attribute: attrs.get(attribute) for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) @@ -411,7 +407,7 @@ def test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_tra assert response.json()["choices"][0]["message"]["content"] == "langfuse-echo" attrs: Final = _observation_span(rig.collector, response.json()["id"]) - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) def test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse( @@ -440,11 +436,7 @@ def test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfus assert response.status_code == 200, response.text attrs: Final = _observation_span(rig.collector, response.json()["id"]) - assert "langfuse.observation.metadata" in attrs, sorted(attrs) - observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) - assert not [name for name in observation if name.startswith("user_api_key")], observation - assert not [key_ for key_, value in observation.items() if value is None], observation - assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation + assert "langfuse.observation.metadata" not in attrs, attrs["langfuse.observation.metadata"] assert not [name for name in attrs if name.startswith("langfuse.trace.metadata.")], sorted(attrs) @@ -472,7 +464,7 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream(gateway: assert chunks, "stream produced no chunks" response_id: Final = chunks[-1].id attrs: Final = _observation_span(rig.collector, response_id) - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) async def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async(gateway: Gateway, tmp_path: Path) -> None: @@ -494,7 +486,7 @@ async def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async(gate extra_body={"cache": {"no-cache": True}}, ) attrs: Final = _observation_span(rig.collector, response.id) - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages(gateway: Gateway, tmp_path: Path) -> None: @@ -517,7 +509,7 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages(gate extra_body={"litellm_metadata": {"tags": ["lf8283"]}, "metadata": {"user_id": identity["end_user"]}}, ) attrs: Final = _observation_span(rig.collector, message.id) - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) async def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream( @@ -545,7 +537,7 @@ async def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream( ): final: Final = await stream.get_final_message() attrs: Final = _observation_span(rig.collector, final.id) - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) def test_langfuse_otel_metadata_reaches_langfuse_for_responses_api(gateway: Gateway, tmp_path: Path) -> None: @@ -567,7 +559,7 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_api(gateway: Gate extra_body={"cache": {"no-cache": True}}, ) attrs: Final = _observation_span(rig.collector, response.id) - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gateway: Gateway, tmp_path: Path) -> None: @@ -598,7 +590,7 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gatewa event["response_id"] for event in events if event.get("type") == "response.output_text.delta" ) attrs: Final = _observation_span(rig.collector, response_id) - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) def test_langfuse_otel_caller_trace_metadata_survives_flattened_identity(gateway: Gateway, tmp_path: Path) -> None: @@ -623,7 +615,7 @@ def test_langfuse_otel_caller_trace_metadata_survives_flattened_identity(gateway assert response.status_code == 200, response.text attrs: Final = _observation_span(rig.collector, response.json()["id"]) assert json.loads(str(attrs["langfuse.trace.metadata"])) == {"deploy": "blue"}, attrs - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) def test_langfuse_otel_failure_span_carries_request_metadata(gateway: Gateway, tmp_path: Path) -> None: @@ -646,7 +638,7 @@ def test_langfuse_otel_failure_span_carries_request_metadata(gateway: Gateway, t ) assert response.status_code >= 500, response.text attrs: Final = _marked_span(rig.collector, marker, "langfuse.observation.") - _assert_identity(attrs, _chat_identity_fields(identity), spend_logs_metadata={"ticket": "LIT-8283"}) + _assert_identity(attrs, _chat_identity_fields(identity)) def test_langfuse_otel_teamless_key_emits_no_team_identity(gateway: Gateway, tmp_path: Path) -> None: @@ -672,7 +664,7 @@ def test_langfuse_otel_teamless_key_emits_no_team_identity(gateway: Gateway, tmp observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) for field in ("user_api_key_team_id", "user_api_key_team_alias", "user_api_key_end_user_id"): assert field not in observation, observation - assert not [key_ for key_, value in observation.items() if value is None], observation + assert set(observation) <= _EXPORTED_FIELDS, observation assert { attribute: attrs.get(attribute) for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) @@ -706,7 +698,7 @@ def test_langfuse_otel_teamless_key_keeps_caller_trace_metadata(gateway: Gateway assert attrs.get("langfuse.trace.metadata.user_api_key_alias") == identity["key_alias"], attrs -def test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_values( +def test_langfuse_otel_oversized_caller_metadata_is_not_copied_into_the_langfuse_keys( gateway: Gateway, tmp_path: Path ) -> None: marker: Final = "lf-bigmeta-" + uuid.uuid4().hex @@ -716,6 +708,7 @@ def test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_va "steps": ["a", "b"], "nested": {"inner": {"leaf": True}}, } + caller_note: Final = "y" * 50000 with ( _langfuse_rig(gateway, tmp_path, {}, marker) as rig, rig.candidate.scenario() as scenario, @@ -728,15 +721,17 @@ def test_langfuse_otel_spend_logs_metadata_roundtrips_oversized_and_nonstring_va "model": identity["model"], "messages": [{"role": "user", "content": marker}], "user": identity["end_user"], + "metadata": {"caller_note": caller_note}, "cache": {"no-cache": True}, }, key=key, ) assert response.status_code == 200, response.text attrs: Final = _observation_span(rig.collector, response.json()["id"]) - observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) - assert observation["spend_logs_metadata"] == dict(spend_logs_metadata), observation _assert_identity(attrs, _chat_identity_fields(identity)) + assert len(str(attrs["langfuse.observation.metadata"])) < 1000, len(str(attrs["langfuse.observation.metadata"])) + assert caller_note in str(attrs["metadata"]), "the bare metadata attribute keeps the caller metadata" + assert spend_logs_metadata["blob"] in str(attrs["metadata"]), "the bare metadata attribute keeps it" def test_langfuse_otel_unauthenticated_request_leaks_no_identity(gateway: Gateway, tmp_path: Path) -> None: @@ -1041,8 +1036,7 @@ def test_langfuse_otel_v2_mapper_emits_request_metadata_and_identity(gateway: Ga observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) expected: Final = _chat_identity_fields(identity) assert {key: observation.get(key) for key in expected} == dict(expected), observation - assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation - assert not [key_ for key_, value in observation.items() if value is None], observation + assert set(observation) <= _EXPORTED_FIELDS, observation flattened: Final = { attribute: attrs.get(attribute) for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) @@ -1086,11 +1080,7 @@ def test_langfuse_otel_v2_mapper_redacts_user_api_key_fields(gateway: Gateway, t seconds=25, ) attrs: Final = next(s for s in observed if s.get("gen_ai.response.id") == response.json()["id"]) - assert "langfuse.observation.metadata" in attrs, sorted(attrs) - observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) - assert not [name for name in observation if name.startswith("user_api_key")], observation - assert not [key_ for key_, value in observation.items() if value is None], observation - assert observation["spend_logs_metadata"] == {"ticket": "LIT-8283"}, observation + assert "langfuse.observation.metadata" not in attrs, attrs["langfuse.observation.metadata"] assert not [name for name in attrs if name.startswith("langfuse.trace.metadata.user_api_key_")], sorted(attrs) @@ -1129,7 +1119,7 @@ def test_langfuse_otel_v2_mapper_teamless_key_emits_no_team_identity(gateway: Ga observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) for field in ("user_api_key_team_id", "user_api_key_team_alias", "user_api_key_end_user_id"): assert field not in observation, observation - assert not [key_ for key_, value in observation.items() if value is None], observation + assert set(observation) <= _EXPORTED_FIELDS, observation assert { attribute: attrs.get(attribute) for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) @@ -1185,7 +1175,7 @@ def test_otel_v2_scalar_requester_metadata_keeps_the_llm_span( assert attrs.get("gen_ai.usage.input_tokens") == 3, attrs if callback == "langfuse_otel": observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) - assert observation["requester_metadata"] == 1, observation + assert "requester_metadata" not in observation, observation assert observation["user_api_key_alias"] == identity["key_alias"], observation @@ -1205,7 +1195,7 @@ def test_langfuse_otel_scalar_requester_metadata_keeps_the_observation_input(gat attrs: Final = _observation_span(rig.collector, response.json()["id"]) assert marker in str(attrs.get("langfuse.observation.input")), sorted(attrs) observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) - assert observation["requester_metadata"] == 1, observation + assert "requester_metadata" not in observation, observation assert observation["user_api_key_alias"] == identity["key_alias"], observation @@ -1249,6 +1239,7 @@ def test_otel_v2_collector_beside_langfuse_gets_no_langfuse_request_metadata( observation: Final = json.loads(str(langfuse["langfuse.observation.metadata"])) assert observation["user_api_key_alias"] == identity["key_alias"], observation assert langfuse["langfuse.trace.metadata.user_api_key_alias"] == identity["key_alias"], langfuse + assert probe not in json.dumps(langfuse, default=str), langfuse collector: Final = _v2_llm_span(generic, response.json()["id"]) assert "langfuse.observation.metadata" not in collector, sorted(collector) @@ -1260,7 +1251,71 @@ def test_otel_v2_collector_beside_langfuse_gets_no_langfuse_request_metadata( assert collector.get("gen_ai.usage.input_tokens") == 3, collector -_PROMPT_QUOTING_ROUTING_FIELDS: Final = ("signals", "matched_keyword", "escalation_keyword", "classifier_crux") +@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/messages", "/v1/responses"]) +def test_otel_v2_langfuse_gets_no_request_headers_or_caller_metadata( + route: str, gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-v2-probe-" + uuid.uuid4().hex + probe: Final = "lf-probe-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker, overrides={"LITELLM_OTEL_V2": "1"}) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + model: Final = ( + scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) + if route == "/v1/messages" + else identity["model"] + ) + body: Final = { + "/v1/chat/completions": { + "model": model, + "messages": [{"role": "user", "content": marker}], + "metadata": {"caller_note": probe}, + "cache": {"no-cache": True}, + }, + "/v1/messages": { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": marker}], + "litellm_metadata": {"caller_note": probe}, + }, + "/v1/responses": {"model": model, "input": marker, "litellm_metadata": {"caller_note": probe}}, + }[route] + watch: Final = _watched_spans(rig.collector) + response: Final = rig.candidate.request("POST", route, body, key=key, headers={"x-lit8283-probe": probe}) + assert response.status_code == 200, response.text + settle: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert settle.status_code == 200, settle.text + + def llm_span(spans: Sequence[Mapping[str, object]], response_id: str) -> Mapping[str, object] | None: + return next( + (span for span in spans if _response_id(span.get("gen_ai.response.id")) == _response_id(response_id)), + None, + ) + + exported: Final = eventually( + watch, + lambda spans: ( + llm_span(spans, response.json()["id"]) is not None and llm_span(spans, settle.json()["id"]) is not None + ), + seconds=25, + ) + attrs: Final = llm_span(exported, response.json()["id"]) + assert attrs is not None + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + assert observation["user_api_key_alias"] == identity["key_alias"], observation + assert set(observation) <= _EXPORTED_FIELDS, observation + assert not [span for span in exported if probe in json.dumps(span, default=str)], exported def _keyword_router(router: str, keyword: str) -> Callable[[str], Sequence[Mapping[str, object]]]: @@ -1331,9 +1386,13 @@ def test_langfuse_otel_metadata_omits_prompt_quoting_routing_fields( if otel_v2 else _observation_span(rig.collector, response.json()["id"]) ) - routing: Final = json.loads(str(attrs["langfuse.observation.metadata"]))["routing_decision"] - assert routing["tier"] == "REASONING", routing - assert routing["router_model_name"] == router, routing - assert not [field for field in _PROMPT_QUOTING_ROUTING_FIELDS if field in routing], routing + observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) + assert set(observation) <= _EXPORTED_FIELDS, observation + langfuse_metadata: Final = { + name: value + for name, value in attrs.items() + if name == "langfuse.observation.metadata" or name.startswith("langfuse.trace.metadata.") + } + assert keyword not in json.dumps(langfuse_metadata, default=str), langfuse_metadata if otel_v2: assert keyword not in json.dumps(attrs, default=str), attrs diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index f33e9dc0f40..9bfa5499542 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -21,6 +21,7 @@ from litellm.integrations.otel.mappers import ( resolve_mappers, ) from litellm.integrations.otel.mappers.openinference import fit_indexed_messages +from litellm.integrations.otel.model.baggage import DEFAULT_BAGGAGE_METADATA_KEYS from litellm.integrations.otel.model.payloads import ( EmbeddingOutput, LLMCallSpanData, @@ -777,33 +778,40 @@ def test_resolve_mappers_rejects_unknown_name(): resolve_mappers(["genai", "nope"]) -def test_langfuse_mapper_emits_request_metadata_and_identity(): +def _langfuse_attrs( + metadata: Mapping[str, object], metadata_keys: tuple[str, ...] = DEFAULT_BAGGAGE_METADATA_KEYS +) -> Mapping[str, object]: payload: Final[dict[str, object]] = { "call_type": "acompletion", "custom_llm_provider": "openai", "model": "gpt-4o", - "metadata": { + "metadata": metadata, + "response": {}, + } + return LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload, metadata_keys=metadata_keys)) + + +def test_langfuse_mapper_emits_request_metadata_and_identity(): + attrs: Final = _langfuse_attrs( + { "user_api_key_alias": "prod-key", "user_api_key_user_id": "user-1", "user_api_key_end_user_id": "end-1", "user_api_key_team_id": "team-9", "user_api_key_team_alias": "team nine", "user_api_key_max_budget": None, + "requester_ip_address": "10.0.0.1", "spend_logs_metadata": {"ticket": "LIT-8283"}, "requester_metadata": {"headers": {"x-tenant": "acme"}}, - }, - "response": {}, - } - attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + } + ) assert json.loads(attrs["langfuse.observation.metadata"]) == { - "user_api_key_alias": "prod-key", - "user_api_key_user_id": "user-1", - "user_api_key_end_user_id": "end-1", "user_api_key_team_id": "team-9", "user_api_key_team_alias": "team nine", - "spend_logs_metadata": {"ticket": "LIT-8283"}, - "requester_metadata": {"headers": {"x-tenant": "acme"}}, + "user_api_key_user_id": "user-1", + "user_api_key_alias": "prod-key", + "user_api_key_end_user_id": "end-1", } assert attrs["langfuse.trace.metadata.user_api_key_alias"] == "prod-key" assert attrs["langfuse.trace.metadata.user_api_key_user_id"] == "user-1" @@ -815,69 +823,69 @@ def test_langfuse_mapper_emits_request_metadata_and_identity(): assert attrs["langfuse.observation.metadata.provider"] == "openai" -def test_langfuse_mapper_omits_prompt_mcp_and_vector_store_content(): - payload: Final[dict[str, object]] = { - "call_type": "acompletion", - "custom_llm_provider": "openai", - "model": "gpt-4o", - "metadata": { - "prompt_management_metadata": { - "prompt_id": "prompt-1", - "prompt_variables": {"input": "prompt-variables-secret"}, - "prompt_integration": "prompt-service", - }, - "mcp_tool_call_metadata": { - "name": "lookup_weather", - "mcp_server_name": "weather-server", - "arguments": {"location": "mcp-arguments-secret"}, - "result": {"forecast": "mcp-result-secret"}, - }, - "vector_store_request_metadata": [ - { - "vector_store_id": "vs-1", - "custom_llm_provider": "openai", - "query": "vector-query-secret", - "vector_store_search_response": {"text": "vector-response-secret"}, - "start_time": 1.0, - "end_time": 2.0, - } - ], - }, - "response": {}, - } - attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) - observation_metadata: Final = attrs["langfuse.observation.metadata"] - serialized: Final = json.dumps(observation_metadata) - parsed: Final = json.loads(observation_metadata) +def test_langfuse_mapper_exports_only_the_promoted_metadata_and_the_team(): + attrs: Final = _langfuse_attrs( + { + "user_api_key_alias": "prod-key", + "user_api_key_team_id": "team-9", + "requester_ip_address": "10.0.0.1", + "user_api_key_user_email": "caller-email-secret", + "user_api_key_auth_metadata": {"note": "auth-metadata-secret"}, + "user_agent": "user-agent-secret", + "headers": {"x-tenant-secret": "header-secret"}, + "requester_custom_headers": {"x-tenant-secret": "custom-header-secret"}, + "requester_metadata": {"note": "requester-metadata-secret"}, + "spend_logs_metadata": {"note": "spend-logs-secret"}, + "prompt_management_metadata": {"prompt_variables": {"input": "prompt-variables-secret"}}, + "mcp_tool_call_metadata": {"arguments": {"location": "mcp-arguments-secret"}}, + "vector_store_request_metadata": [{"query": "vector-query-secret"}], + "routing_decision": {"matched_keyword": "routing-prompt-secret"}, + } + ) - assert "prompt-variables-secret" not in serialized - assert "mcp-arguments-secret" not in serialized - assert "mcp-result-secret" not in serialized - assert "vector-query-secret" not in serialized - assert "vector-response-secret" not in serialized - assert parsed["prompt_management_metadata"]["prompt_id"] == "prompt-1" - assert parsed["prompt_management_metadata"]["prompt_integration"] == "prompt-service" - assert parsed["mcp_tool_call_metadata"]["name"] == "lookup_weather" - assert parsed["mcp_tool_call_metadata"]["mcp_server_name"] == "weather-server" - assert parsed["vector_store_request_metadata"][0]["vector_store_id"] == "vs-1" - assert parsed["vector_store_request_metadata"][0]["custom_llm_provider"] == "openai" - assert parsed["vector_store_request_metadata"][0]["start_time"] == 1.0 - assert parsed["vector_store_request_metadata"][0]["end_time"] == 2.0 + assert "secret" not in json.dumps(attrs) + assert json.loads(attrs["langfuse.observation.metadata"]) == { + "user_api_key_team_id": "team-9", + "user_api_key_alias": "prod-key", + } + + +@pytest.mark.parametrize( + ("metadata_keys", "expected"), + [ + ((), {"user_api_key_team_id": "team-9"}), + (("user_api_key_user_id",), {"user_api_key_team_id": "team-9", "user_api_key_user_id": "user-1"}), + (("requester_metadata.trace_id", "requester_ip_address"), {"user_api_key_team_id": "team-9"}), + ], +) +def test_langfuse_mapper_follows_the_operator_metadata_allowlist( + metadata_keys: tuple[str, ...], expected: Mapping[str, str] +) -> None: + attrs: Final = _langfuse_attrs( + { + "user_api_key_alias": "prod-key", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "team-9", + "requester_ip_address": "10.0.0.1", + "requester_metadata": {"trace_id": "trace-7", "note": "requester-metadata-secret"}, + }, + metadata_keys=metadata_keys, + ) + + assert json.loads(attrs["langfuse.observation.metadata"]) == expected + assert {key for key in attrs if key.startswith("langfuse.trace.metadata.user_api_key_")} == { + f"langfuse.trace.metadata.{key}" for key in expected if key.startswith("user_api_key_") + } def test_langfuse_mapper_skips_empty_identity_values(): - payload: Final[dict[str, object]] = { - "call_type": "acompletion", - "custom_llm_provider": "openai", - "model": "gpt-4o", - "metadata": { + attrs: Final = _langfuse_attrs( + { "user_api_key_alias": None, "user_api_key_user_id": "", "user_api_key_end_user_id": "end-1", - }, - "response": {}, - } - attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + } + ) assert "langfuse.trace.metadata.user_api_key_alias" not in attrs assert "langfuse.trace.metadata.user_api_key_user_id" not in attrs @@ -888,37 +896,23 @@ def test_langfuse_mapper_redacts_user_api_key_fields(monkeypatch: pytest.MonkeyP import litellm monkeypatch.setattr(litellm, "redact_user_api_key_info", True) - payload: Final[dict[str, object]] = { - "call_type": "acompletion", - "custom_llm_provider": "openai", - "model": "gpt-4o", - "metadata": { + attrs: Final = _langfuse_attrs( + { "user_api_key_alias": "prod-key", "user_api_key_team_id": "team-9", - "spend_logs_metadata": {"ticket": "LIT-8283"}, - }, - "response": {}, - } - attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + "requester_ip_address": "10.0.0.1", + } + ) - exported: Final = json.loads(attrs["langfuse.observation.metadata"]) - assert not [key for key in exported if key.startswith("user_api_key")] - assert exported["spend_logs_metadata"] == {"ticket": "LIT-8283"} + assert "langfuse.observation.metadata" not in attrs assert not [key for key in attrs if key.startswith("langfuse.trace.metadata.user_api_key_")] assert attrs["langfuse.trace.metadata.team_id"] == "team-9" def test_langfuse_mapper_omits_observation_metadata_without_payload_metadata(): - payload: Final[dict[str, object]] = { - "call_type": "acompletion", - "custom_llm_provider": "openai", - "model": "gpt-4o", - "metadata": {}, - "response": {}, - } - attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + attrs: Final = _langfuse_attrs({}) - assert json.loads(attrs["langfuse.observation.metadata"]) == {"requester_metadata": {}} + assert "langfuse.observation.metadata" not in attrs @pytest.mark.parametrize( @@ -933,43 +927,6 @@ def test_langfuse_mapper_omits_observation_metadata_without_payload_metadata(): ], ) def test_span_data_builds_from_request_metadata_of_unexpected_shape(unexpected_shape: dict[str, object]) -> None: - payload: Final[dict[str, object]] = { - "call_type": "acompletion", - "custom_llm_provider": "openai", - "model": "gpt-4o", - "metadata": {**unexpected_shape, "spend_logs_metadata": {"ticket": "LIT-8283"}}, - "response": {}, - } - attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) + attrs: Final = _langfuse_attrs({**unexpected_shape, "user_api_key_alias": "prod-key"}) - assert json.loads(attrs["langfuse.observation.metadata"])["spend_logs_metadata"] == {"ticket": "LIT-8283"} - - -def test_langfuse_mapper_omits_prompt_quoting_routing_fields(): - payload: Final[dict[str, object]] = { - "call_type": "acompletion", - "custom_llm_provider": "openai", - "model": "gpt-4o", - "metadata": { - "routing_decision": { - "router_model_name": "smart-router", - "router_type": "complexity", - "cause": "literal_keyword_match", - "tier": "REASONING", - "signals": ["keyword (routing-prompt-secret)"], - "matched_keyword": "routing-prompt-secret", - "escalation_keyword": "routing-prompt-secret", - "classifier_crux": "routing-prompt-secret", - } - }, - "response": {}, - } - attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload)) - - assert "routing-prompt-secret" not in json.dumps(attrs) - assert json.loads(attrs["langfuse.observation.metadata"])["routing_decision"] == { - "router_model_name": "smart-router", - "router_type": "complexity", - "cause": "literal_keyword_match", - "tier": "REASONING", - } + assert json.loads(attrs["langfuse.observation.metadata"]) == {"user_api_key_alias": "prod-key"} diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 19215fe3fc4..8b5332db190 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -496,6 +496,7 @@ class TestLangfuseOtelIntegration: "user_api_key_end_user_id": "end-user-1", "user_api_key_team_id": "team-1", "user_api_key_team_alias": "team-a", + "user_api_key_org_id": "org-1", "spend_logs_metadata": {"env": "prod"}, "requester_ip_address": "10.0.0.1", "requester_metadata": {"ticket": "LIT-8283"}, @@ -510,7 +511,14 @@ class TestLangfuseOtelIntegration: actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - assert json.loads(actual["langfuse.observation.metadata"]) == {**request_metadata} + assert json.loads(actual["langfuse.observation.metadata"]) == { + "user_api_key_alias": "prod-key", + "user_api_key_user_id": "user-1", + "user_api_key_end_user_id": "end-user-1", + "user_api_key_team_id": "team-1", + "user_api_key_team_alias": "team-a", + "user_api_key_org_id": "org-1", + } assert {key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.")} == { "langfuse.trace.metadata.user_api_key_alias": "prod-key", "langfuse.trace.metadata.user_api_key_user_id": "user-1", @@ -542,36 +550,24 @@ class TestLangfuseOtelIntegration: actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - assert json.loads(actual["langfuse.observation.metadata"]) == { - "spend_logs_metadata": {"env": "prod"}, - "requester_ip_address": "10.0.0.1", - "requester_metadata": {"ticket": "LIT-8283"}, - } + assert "langfuse.observation.metadata" not in actual assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] - def test_request_metadata_omits_prompt_mcp_and_vector_store_content(self) -> None: + def test_request_metadata_beyond_the_identity_fields_stays_off_the_langfuse_metadata_keys(self) -> None: request_metadata: Final = { - "prompt_management_metadata": { - "prompt_id": "prompt-1", - "prompt_variables": {"input": "prompt-variables-secret"}, - "prompt_integration": "prompt-service", - }, - "mcp_tool_call_metadata": { - "name": "lookup_weather", - "mcp_server_name": "weather-server", - "arguments": {"location": "mcp-arguments-secret"}, - "result": {"forecast": "mcp-result-secret"}, - }, - "vector_store_request_metadata": [ - { - "vector_store_id": "vs-1", - "custom_llm_provider": "openai", - "query": "vector-query-secret", - "vector_store_search_response": {"text": "vector-response-secret"}, - "start_time": 1.0, - "end_time": 2.0, - } - ], + "user_api_key_alias": "prod-key", + "requester_ip_address": "10.0.0.1", + "user_api_key_user_email": "caller-email-secret", + "user_api_key_auth_metadata": {"note": "auth-metadata-secret"}, + "user_agent": "user-agent-secret", + "headers": {"x-tenant-secret": "header-secret"}, + "requester_custom_headers": {"x-tenant-secret": "custom-header-secret"}, + "requester_metadata": {"note": "requester-metadata-secret"}, + "spend_logs_metadata": {"note": "spend-logs-secret"}, + "prompt_management_metadata": {"prompt_variables": {"input": "prompt-variables-secret"}}, + "mcp_tool_call_metadata": {"arguments": {"location": "mcp-arguments-secret"}}, + "vector_store_request_metadata": [{"query": "vector-query-secret"}], + "routing_decision": {"matched_keyword": "routing-prompt-secret"}, } kwargs: Final[dict[str, object]] = { "litellm_params": {"metadata": {}}, @@ -582,25 +578,18 @@ class TestLangfuseOtelIntegration: LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - observation_metadata: Final = actual["langfuse.observation.metadata"] - serialized: Final = json.dumps(observation_metadata) - parsed: Final = json.loads(observation_metadata) + langfuse_metadata: Final = { + key: value + for key, value in actual.items() + if key == "langfuse.observation.metadata" or key.startswith("langfuse.trace.metadata.") + } - assert "prompt-variables-secret" not in serialized - assert "mcp-arguments-secret" not in serialized - assert "mcp-result-secret" not in serialized - assert "vector-query-secret" not in serialized - assert "vector-response-secret" not in serialized - assert parsed["prompt_management_metadata"]["prompt_id"] == "prompt-1" - assert parsed["prompt_management_metadata"]["prompt_integration"] == "prompt-service" - assert parsed["mcp_tool_call_metadata"]["name"] == "lookup_weather" - assert parsed["mcp_tool_call_metadata"]["mcp_server_name"] == "weather-server" - assert parsed["vector_store_request_metadata"][0]["vector_store_id"] == "vs-1" - assert parsed["vector_store_request_metadata"][0]["custom_llm_provider"] == "openai" - assert parsed["vector_store_request_metadata"][0]["start_time"] == 1.0 - assert parsed["vector_store_request_metadata"][0]["end_time"] == 2.0 + assert langfuse_metadata == { + "langfuse.observation.metadata": json.dumps({"user_api_key_alias": "prod-key"}), + "langfuse.trace.metadata.user_api_key_alias": "prod-key", + } - def test_request_metadata_drops_null_fields_and_skips_empty_trace_identities(self) -> None: + def test_request_metadata_drops_empty_fields_and_skips_empty_trace_identities(self) -> None: request_metadata: Final = { "user_api_key_alias": "prod-key", "user_api_key_user_id": "user-1", @@ -624,15 +613,28 @@ class TestLangfuseOtelIntegration: assert json.loads(actual["langfuse.observation.metadata"]) == { "user_api_key_alias": "prod-key", "user_api_key_user_id": "user-1", - "user_api_key_team_id": "", - "spend_logs_metadata": {"env": "prod"}, - "requester_metadata": {}, } assert {key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.")} == { "langfuse.trace.metadata.user_api_key_alias": "prod-key", "langfuse.trace.metadata.user_api_key_user_id": "user-1", } + def test_request_metadata_keys_are_absent_without_identity_fields(self) -> None: + kwargs: Final[dict[str, object]] = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": { + "metadata": {"spend_logs_metadata": {"env": "prod"}, "requester_ip_address": "10.0.0.1"} + }, + } + + with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + + actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} + + assert "langfuse.observation.metadata" not in actual + assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] + def test_request_metadata_keys_are_absent_without_standard_logging_object(self) -> None: kwargs: Final = {"litellm_params": {"metadata": {"trace_metadata": {"k": "v"}}}} @@ -653,14 +655,17 @@ class TestLangfuseOtelIntegration: {"requester_metadata": 1}, {"requester_metadata": True}, {"requester_metadata": 1.5}, + {1: "one"}, + {"user_api_key_user_id": 5}, + {"user_api_key_team_id": {"id": "team-1"}}, ], ) def test_request_metadata_of_unexpected_shape_keeps_the_observation_output( - self, unexpected_shape: dict[str, object] + self, unexpected_shape: dict[object, object] ) -> None: kwargs: Final[dict[str, object]] = { "litellm_params": {"metadata": {}}, - "standard_logging_object": {"metadata": {**unexpected_shape, "spend_logs_metadata": {"env": "prod"}}}, + "standard_logging_object": {"metadata": {**unexpected_shape, "user_api_key_alias": "prod-key"}}, } response_obj: Final = {"choices": [{"message": {"role": "assistant", "content": "hello"}}]} @@ -669,36 +674,9 @@ class TestLangfuseOtelIntegration: actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - assert json.loads(actual["langfuse.observation.metadata"])["spend_logs_metadata"] == {"env": "prod"} + assert json.loads(actual["langfuse.observation.metadata"]) == {"user_api_key_alias": "prod-key"} assert json.loads(actual["langfuse.observation.output"]) == {"role": "assistant", "content": "hello"} - def test_observation_metadata_omits_prompt_quoting_routing_fields(self) -> None: - kwargs: Final[dict[str, object]] = { - "litellm_params": {"metadata": {}}, - "standard_logging_object": { - "metadata": { - "routing_decision": { - "router_type": "complexity", - "tier": "REASONING", - "signals": ["keyword (routing-prompt-secret)"], - "matched_keyword": "routing-prompt-secret", - "escalation_keyword": "routing-prompt-secret", - "classifier_crux": "routing-prompt-secret", - } - } - }, - } - - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) - - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - - assert json.loads(actual["langfuse.observation.metadata"])["routing_decision"] == { - "router_type": "complexity", - "tier": "REASONING", - } - class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target From 3d3e76ce81581850caf9904d4888803103aa1010 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 19 Oct 2026 00:43:19 -0700 Subject: [PATCH 24/28] fix(langfuse_otel): drop the caller identity first when a span hits its attribute limit v1 wrote the identity attributes after the OpenInference ones, so a span near the 128-attribute limit (about 33+ tools) evicted openinference.span.kind, metadata and llm.model_name instead. The identity now goes first on a fresh span, so the SDK evicts it before anything else, and a caller's active span that already carries attributes does not get it. v2 sheds the Langfuse identity keys before any indexed message attribute when a span is over budget, so a span keeps every attribute it carried without them. --- .../integrations/langfuse/langfuse_otel.py | 8 +- litellm/integrations/otel/emitter.py | 5 +- .../otel/mappers/openinference.py | 10 +- .../test_langfuse_otel_metadata.py | 154 ++++++++++++++++++ .../integrations/otel/test_otel_v2_emitter.py | 31 ++++ tests/unit/integrations/test_langfuse_otel.py | 125 ++++++++++++-- 6 files changed, 313 insertions(+), 20 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index a8651bc2555..3efa474b6db 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -51,7 +51,14 @@ class LangfuseOtelLogger(OpenTelemetry): Sets OpenTelemetry span attributes for Langfuse observability. Uses the same attribute setting logic as Arize Phoenix for consistency. """ + from opentelemetry.sdk.trace import ReadableSpan + # The caller identity goes first, so a span that reaches its attribute limit evicts it before any + # other attribute. A span that already carries attributes (a caller's active span) does not get it. + if not (isinstance(span, ReadableSpan) and span.attributes): + LangfuseOtelLogger._set_request_metadata_attributes( + span=span, kwargs=_MAPPING_ADAPTER.validate_python(kwargs) + ) _utils.set_attributes(span, kwargs, response_obj, LangfuseLLMObsOTELAttributes) span.set_attribute("langfuse.observation.type", "generation") @@ -278,7 +285,6 @@ class LangfuseOtelLogger(OpenTelemetry): metadata: Final = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata) - LangfuseOtelLogger._set_request_metadata_attributes(span=span, kwargs=_MAPPING_ADAPTER.validate_python(kwargs)) messages: Final = kwargs.get("messages") if messages: diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index a4cb0a84541..225ea87f078 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -13,6 +13,7 @@ from opentelemetry.trace.status import Status, StatusCode from litellm.integrations.otel.mappers import resolve_mappers from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData +from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES from litellm.integrations.otel.mappers.openinference import fit_indexed_messages from litellm.integrations.otel.model.config import OpenTelemetryV2Config from litellm.integrations.otel.model.payloads import ( @@ -303,7 +304,9 @@ class SpanEmitter: ) stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES reserved: Final = len(stamped_later.keys() - mapped.keys()) - for key, value in fit_indexed_messages(mapped, attribute_budget(span, reserved, frozenset(mapped))).items(): + # The Langfuse request metadata sheds first, so a span at its attribute limit loses nothing else to it. + budget: Final = attribute_budget(span, reserved, frozenset(mapped)) + for key, value in fit_indexed_messages(mapped, budget, LANGFUSE_REQUEST_METADATA_ATTRIBUTES).items(): span.set_attribute(key, value) if error: stamped: Final = stamp_error(span, error) diff --git a/litellm/integrations/otel/mappers/openinference.py b/litellm/integrations/otel/mappers/openinference.py index 8d7b4d7cf4a..0c465bbe5cb 100644 --- a/litellm/integrations/otel/mappers/openinference.py +++ b/litellm/integrations/otel/mappers/openinference.py @@ -153,7 +153,9 @@ def _shed_order(groups: Mapping[tuple[str, int, int], tuple[str, ...]]) -> tuple _METADATA_KEY: Final = "metadata" -def fit_indexed_messages(attrs: Mapping[str, AttrValue], budget: int | None) -> Mapping[str, AttrValue]: +def fit_indexed_messages( + attrs: Mapping[str, AttrValue], budget: int | None, shed_first: frozenset[str] = frozenset() +) -> Mapping[str, AttrValue]: """``attrs`` with indexed message attributes shed, least valuable first, until at most ``budget`` keys remain. ``None`` means the span has no attribute count limit. Every message still rides the ``input.value`` and @@ -161,12 +163,14 @@ def fit_indexed_messages(attrs: Mapping[str, AttrValue], budget: int | None) -> after every indexed message attribute: message attributes are the indexed, queryable view (the blobs carry no per-index keys), so a count-squeezed span keeps them and the single metadata key absorbs only the residual shortfall. Shedding happens here, before ``span.set_attribute``, so the fit is exact and - the SDK's dropped-attributes counter never silently masks the choice. + the SDK's dropped-attributes counter never silently masks the choice. ``shed_first`` keys shed one at a + time before anything else, so they never cost the span any other attribute. """ if budget is None or len(attrs) <= budget: return attrs groups: Final = _message_key_groups(attrs) - sheddable: Final = (*(groups[group] for group in _shed_order(groups)),) + leading: Final = tuple((key,) for key in attrs if key in shed_first) + sheddable: Final = (*leading, *(groups[group] for group in _shed_order(groups))) flex: Final = (_METADATA_KEY,) if _METADATA_KEY in attrs else () candidates: Final = (*sheddable, flex) if flex else sheddable running: Final = tuple(accumulate(len(candidate) for candidate in candidates)) diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index be867d07fe6..d43ff4779bb 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -1396,3 +1396,157 @@ def test_langfuse_otel_metadata_omits_prompt_quoting_routing_fields( assert keyword not in json.dumps(langfuse_metadata, default=str), langfuse_metadata if otel_v2: assert keyword not in json.dumps(attrs, default=str), attrs + + +def _span_records(body: bytes) -> tuple[tuple[dict[str, object], int], ...]: + request: Final = ExportTraceServiceRequest.FromString(body) + return tuple( + ( + { + attribute.key: getattr(attribute.value, attribute.value.WhichOneof("value")) + for attribute in span.attributes + }, + span.dropped_attributes_count, + ) + for resource in request.resource_spans + for scope in resource.scope_spans + for span in scope.spans + ) + + +def _identity_attributes(attrs: Mapping[str, object]) -> frozenset[str]: + names: Final = {"langfuse.observation.metadata", *(f"langfuse.trace.metadata.{field}" for field in _IDENTITIES)} + return frozenset(key for key in attrs if key in names) + + +def _limit_sweep( + rig: _Rig, + key: str, + marker: str, + bodies: Mapping[int, Mapping[str, object]], +) -> dict[int, tuple[dict[str, object], int]]: + """The exported span and dropped-attribute count per size; each body's prompt carries ``::``.""" + drained: list[ + tuple[dict[str, object], int] + ] = [] # mutable-ok: drain() consumes the wire queue, so observed spans must accumulate across eventually polls + for body in bodies.values(): + response = rig.candidate.request("POST", "/v1/chat/completions", dict(body), key=key) + assert response.status_code == 200, response.text + + def by_size() -> dict[int, tuple[dict[str, object], int] | None]: + drained.extend(record for batch in rig.collector.drain() for record in _span_records(batch.body)) + return { + size: next( + ( + (attrs, dropped) + for attrs, dropped in drained + if f"{marker}:{size}:" in str(attrs.get("langfuse.observation.input", "")) + ), + None, + ) + for size in bodies + } + + observed: Final = eventually(by_size, lambda spans: None not in spans.values(), seconds=30) + return {size: record for size, record in observed.items() if record is not None} + + +def test_langfuse_otel_identity_is_evicted_before_any_other_attribute_of_a_full_span( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-limit-" + uuid.uuid4().hex + with ( + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + + def body(tool_count: int) -> dict[str, object]: + return { + "model": identity["model"], + "messages": [{"role": "user", "content": f"{marker}:{tool_count}:"}], + "user": identity["end_user"], + "tools": [ + { + "type": "function", + "function": { + "name": f"tool_{n}", + "description": f"looks up thing {n}", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + }, + } + for n in range(tool_count) + ], + "cache": {"no-cache": True}, + } + + spans: Final = _limit_sweep( + rig, key, marker, {tool_count: body(tool_count) for tool_count in (0, *range(30, 41))} + ) + + reference, reference_dropped = spans[0] + identity_count: Final = len(_identity_attributes(reference)) + assert reference_dropped == 0, reference_dropped + assert identity_count == len(_chat_identity_fields(identity)) + 1, sorted(reference) + for tool_count, (attrs, dropped) in spans.items(): + kept = _identity_attributes(attrs) + assert len(kept) == identity_count - min(dropped, identity_count), (tool_count, dropped, sorted(kept)) + if dropped <= identity_count: + expected = {"openinference.span.kind", "metadata", "llm.model_name", "llm.request.type"} + assert expected <= set(attrs), (tool_count, dropped, sorted(expected - set(attrs))) + tools = {f"llm.tools.{n}.name" for n in range(tool_count)} + assert tools <= set(attrs), (tool_count, dropped, sorted(tools - set(attrs))) + assert [tool_count for tool_count, (_, dropped) in spans.items() if 0 < dropped <= identity_count], { + tool_count: dropped for tool_count, (_, dropped) in spans.items() + } + + +def test_otel_v2_langfuse_identity_sheds_before_any_message_attribute(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "lf-v2-limit-" + uuid.uuid4().hex + with ( + _langfuse_rig( + gateway, + tmp_path, + {}, + marker, + overrides={ + "LITELLM_OTEL_V2": "1", + "MAPPER_NAMES": "genai,openinference", + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "span_only", + }, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + + def body(message_count: int) -> dict[str, object]: + return { + "model": identity["model"], + "messages": [ + {"role": ("user", "assistant")[n % 2], "content": f"{marker} turn {n}"} + for n in range(message_count - 1) + ] + + [{"role": "user", "content": f"{marker}:{message_count}: turn {message_count - 1}"}], + "user": identity["end_user"], + "cache": {"no-cache": True}, + } + + spans: Final = _limit_sweep( + rig, key, marker, {message_count: body(message_count) for message_count in (1, *range(20, 61))} + ) + + reference, _ = spans[1] + identity_count: Final = len(_identity_attributes(reference)) + assert identity_count == len(_chat_identity_fields(identity)) + 1, sorted(reference) + shed_by_size: Final[dict[int, tuple[int, int]]] = {} # mutable-ok: filled once per swept span below + for message_count, (attrs, _) in spans.items(): + kept = {n for n in range(message_count) if f"llm.input_messages.{n}.message.content" in attrs} + shed_messages = message_count - len(kept) + shed_identity = identity_count - len(_identity_attributes(attrs)) + shed_by_size[message_count] = (shed_identity, shed_messages) + if shed_messages: + assert shed_identity == identity_count, (message_count, shed_identity, shed_messages) + assert [ + size for size, (shed_identity, shed_messages) in shed_by_size.items() if shed_identity and not shed_messages + ], shed_by_size + assert [size for size, (_, shed_messages) in shed_by_size.items() if shed_messages], shed_by_size diff --git a/tests/unit/integrations/otel/test_otel_v2_emitter.py b/tests/unit/integrations/otel/test_otel_v2_emitter.py index 949eb97bccd..044ecd0c09e 100644 --- a/tests/unit/integrations/otel/test_otel_v2_emitter.py +++ b/tests/unit/integrations/otel/test_otel_v2_emitter.py @@ -795,6 +795,37 @@ def test_indexed_messages_follow_the_span_limits_of_a_per_request_tracer_overrid assert span.attributes["llm.output_messages.0.message.content"] == "reply 0" +def _team_conversation_span(monkeypatch, redact, max_span_attributes): + monkeypatch.setattr("litellm.redact_user_api_key_info", redact) + return _conversation_span( + ["genai", "openinference", "langfuse"], + _conversation_payload(60), + span_limits=SpanLimits(max_span_attributes=max_span_attributes), + ) + + +def test_langfuse_identity_sheds_before_any_attribute_the_span_carries_without_it(monkeypatch): + """At its attribute limit a span drops the Langfuse identity keys first, then exactly what it drops without them.""" + plain = _team_conversation_span(monkeypatch, True, SpanLimits.UNSET).attributes + full = _team_conversation_span(monkeypatch, False, SpanLimits.UNSET).attributes + identity_keys = set(full) - set(plain) + assert identity_keys == { + "langfuse.observation.metadata", + "langfuse.trace.metadata.user_api_key_team_id", + "langfuse.trace.metadata.user_api_key_team_alias", + } + + squeezed_limit = len(plain) - 10 + squeezed_plain = _team_conversation_span(monkeypatch, True, squeezed_limit) + squeezed = _team_conversation_span(monkeypatch, False, squeezed_limit) + assert dict(squeezed.attributes) == dict(squeezed_plain.attributes) + assert squeezed.dropped_attributes == squeezed_plain.dropped_attributes == 0 + + roomy = _team_conversation_span(monkeypatch, False, len(plain) + 2).attributes + assert {key: roomy.get(key) for key in plain} == dict(plain) + assert len(set(roomy) & identity_keys) == 2 + + def test_span_attribute_limit_falls_back_to_the_environment_for_spans_outside_the_sdk(monkeypatch): monkeypatch.setenv("OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", "48") assert span_attribute_limit(INVALID_SPAN) == 48 diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 8b5332db190..eb7630781e4 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -6,7 +6,9 @@ from typing import Final from unittest.mock import MagicMock, patch import pytest -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter import litellm from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger @@ -506,8 +508,11 @@ class TestLangfuseOtelIntegration: "standard_logging_object": {"metadata": request_metadata}, } - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + with ( + patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, + patch("litellm.integrations.arize._utils.set_attributes"), + ): + LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} @@ -545,8 +550,11 @@ class TestLangfuseOtelIntegration: "standard_logging_object": {"metadata": request_metadata}, } - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + with ( + patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, + patch("litellm.integrations.arize._utils.set_attributes"), + ): + LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} @@ -574,8 +582,11 @@ class TestLangfuseOtelIntegration: "standard_logging_object": {"metadata": request_metadata}, } - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + with ( + patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, + patch("litellm.integrations.arize._utils.set_attributes"), + ): + LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} langfuse_metadata: Final = { @@ -605,8 +616,11 @@ class TestLangfuseOtelIntegration: "standard_logging_object": {"metadata": request_metadata}, } - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + with ( + patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, + patch("litellm.integrations.arize._utils.set_attributes"), + ): + LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} @@ -627,8 +641,11 @@ class TestLangfuseOtelIntegration: }, } - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + with ( + patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, + patch("litellm.integrations.arize._utils.set_attributes"), + ): + LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} @@ -638,8 +655,11 @@ class TestLangfuseOtelIntegration: def test_request_metadata_keys_are_absent_without_standard_logging_object(self) -> None: kwargs: Final = {"litellm_params": {"metadata": {"trace_metadata": {"k": "v"}}}} - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) + with ( + patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, + patch("litellm.integrations.arize._utils.set_attributes"), + ): + LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} @@ -669,14 +689,89 @@ class TestLangfuseOtelIntegration: } response_obj: Final = {"choices": [{"message": {"role": "assistant", "content": "hello"}}]} - with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, response_obj) + with ( + patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, + patch("litellm.integrations.arize._utils.set_attributes"), + ): + LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, response_obj) actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} assert json.loads(actual["langfuse.observation.metadata"]) == {"user_api_key_alias": "prod-key"} assert json.loads(actual["langfuse.observation.output"]) == {"role": "assistant", "content": "hello"} + @staticmethod + def _exported_span( + kwargs: dict[str, object], max_span_attributes: int | None, carried: dict[str, str] | None = None + ) -> ReadableSpan: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider(span_limits=SpanLimits(max_span_attributes=max_span_attributes)) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + span: Final = provider.get_tracer("langfuse-otel-test").start_span("litellm_request", attributes=carried) + response_obj: Final = {"id": "chatcmpl-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]} + LangfuseOtelLogger.set_langfuse_otel_attributes(span, kwargs, response_obj) + span.end() + (finished,) = exporter.get_finished_spans() + return finished + + @staticmethod + def _tool_call_kwargs() -> dict[str, object]: + tools: Final = [ + { + "type": "function", + "function": {"name": f"tool_{n}", "description": f"does thing {n}", "parameters": {"type": "object"}}, + } + for n in range(3) + ] + messages: Final = [{"role": "user", "content": "hello"}] + return { + "model": "gpt-4o", + "messages": messages, + "optional_params": {"tools": tools}, + "litellm_params": {"metadata": {}}, + "standard_logging_object": { + "model": "gpt-4o", + "custom_llm_provider": "openai", + "call_type": "completion", + "messages": messages, + "model_parameters": {"tools": tools}, + "metadata": {"user_api_key_alias": "prod-key", "user_api_key_team_id": "team-1"}, + }, + } + + def test_request_metadata_never_evicts_another_attribute_at_the_span_limit( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + kwargs: Final = self._tool_call_kwargs() + monkeypatch.setattr(litellm, "redact_user_api_key_info", True) + plain: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET).attributes or {}) + monkeypatch.setattr(litellm, "redact_user_api_key_info", False) + full: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET).attributes or {}) + identity_keys: Final = set(full) - set(plain) + assert identity_keys == { + "langfuse.observation.metadata", + "langfuse.trace.metadata.user_api_key_alias", + "langfuse.trace.metadata.user_api_key_team_id", + } + + at_limit: Final = self._exported_span(kwargs, len(plain)) + assert dict(at_limit.attributes or {}) == plain + + one_spare: Final = dict(self._exported_span(kwargs, len(plain) + 1).attributes or {}) + assert {key: one_spare.get(key) for key in plain} == plain + assert len(set(one_spare) & identity_keys) == 1 + + def test_request_metadata_skips_a_span_that_already_carries_attributes(self) -> None: + attributes: Final = dict( + self._exported_span(self._tool_call_kwargs(), SpanLimits.UNSET, {"caller.attribute": "kept"}).attributes + or {} + ) + + assert attributes["caller.attribute"] == "kept" + assert attributes["langfuse.observation.type"] == "generation" + assert "langfuse.observation.metadata" not in attributes + assert not [key for key in attributes if key.startswith("langfuse.trace.metadata.")] + class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target From 4fa762baa9b532ace82d9321c98207b572bb6aec Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 19 Oct 2026 11:14:01 -0700 Subject: [PATCH 25/28] fix(langfuse_otel): keep other spans and dropped counts as main exports them - v1 writes the caller identity only into the room the span's attribute limit leaves next to the error attributes a failed call adds, so the exported dropped-attribute count matches main - v2 strips the Langfuse request metadata only from LiteLLM's own spans; application, instrumentation and server spans keep every key they set - a stripped v2 span keeps the dropped attribute, event and link counts - a config that lists the Langfuse mapper without the langfuse_otel exporter also keeps the request metadata away from other exporters --- .../integrations/langfuse/langfuse_otel.py | 58 ++++-- litellm/integrations/otel/logger.py | 2 +- .../integrations/otel/plumbing/providers.py | 80 +++++--- .../test_langfuse_otel_metadata.py | 186 +++++++++++++++++- .../otel/test_otel_v2_destinations.py | 75 +++++-- tests/unit/integrations/test_langfuse_otel.py | 45 ++++- 6 files changed, 372 insertions(+), 74 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 3efa474b6db..6db26c09c32 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -8,11 +8,12 @@ from typing import TYPE_CHECKING, Any, Final, Optional from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.integrations._types.open_inference import ErrorAttributes from litellm.integrations.arize import _utils from litellm.integrations.langfuse.langfuse_otel_attributes import ( LangfuseLLMObsOTELAttributes, ) -from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.integrations.opentelemetry import HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, OpenTelemetry, OpenTelemetryConfig from litellm.integrations.otel.mappers.langfuse import TRACE_IDENTITY_FIELDS from litellm.integrations.otel.model.metadata import IDENTITY_METADATA_FIELDS, exported_request_metadata from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -30,6 +31,17 @@ else: _MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) +# The attributes ``OpenTelemetry._record_exception_on_span`` adds to a failed call's span after these. +_ERROR_ATTRIBUTES: Final = frozenset( + { + ErrorAttributes.ERROR_CODE, + HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, + ErrorAttributes.ERROR_TYPE, + ErrorAttributes.ERROR_MESSAGE, + ErrorAttributes.ERROR_LLM_PROVIDER, + ErrorAttributes.ERROR_STACK_TRACE, + } +) LANGFUSE_CLOUD_EU_ENDPOINT: Final = "https://cloud.langfuse.com/api/public/otel" LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/otel" @@ -53,12 +65,8 @@ class LangfuseOtelLogger(OpenTelemetry): """ from opentelemetry.sdk.trace import ReadableSpan - # The caller identity goes first, so a span that reaches its attribute limit evicts it before any - # other attribute. A span that already carries attributes (a caller's active span) does not get it. - if not (isinstance(span, ReadableSpan) and span.attributes): - LangfuseOtelLogger._set_request_metadata_attributes( - span=span, kwargs=_MAPPING_ADAPTER.validate_python(kwargs) - ) + # A span that already carries attributes (a caller's active span) does not get the caller identity. + fresh: Final = not (isinstance(span, ReadableSpan) and span.attributes) _utils.set_attributes(span, kwargs, response_obj, LangfuseLLMObsOTELAttributes) span.set_attribute("langfuse.observation.type", "generation") @@ -66,6 +74,10 @@ class LangfuseOtelLogger(OpenTelemetry): # Set Langfuse specific attributes ######################################################### LangfuseOtelLogger._set_langfuse_specific_attributes(span=span, kwargs=kwargs, response_obj=response_obj) + if fresh: + LangfuseOtelLogger._set_request_metadata_attributes( + span=span, kwargs=_MAPPING_ADAPTER.validate_python(kwargs) + ) @staticmethod def _extract_langfuse_metadata(kwargs: dict) -> dict: @@ -144,6 +156,10 @@ class LangfuseOtelLogger(OpenTelemetry): span: Span, kwargs: Mapping[object, object], ) -> None: + """Set the caller identity into the room ``span``'s attribute limit leaves next to the error attributes + a failed call adds after it, so the identity never evicts or drops another attribute.""" + from litellm.integrations.otel.emitter import carried_keys, span_attribute_limit + raw_standard_logging_object: Final = kwargs.get("standard_logging_object") if not isinstance(raw_standard_logging_object, dict): return @@ -154,16 +170,26 @@ class LangfuseOtelLogger(OpenTelemetry): observation_metadata: Final = exported_request_metadata( {field: metadata.get(field) for field in IDENTITY_METADATA_FIELDS} ) - if observation_metadata: - _utils.safe_set_attribute( - span, - LangfuseSpanAttributes.OBSERVATION_METADATA.value, - json.dumps(dict(observation_metadata)), - ) trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value - for field in TRACE_IDENTITY_FIELDS: - if value := observation_metadata.get(field): - _utils.safe_set_attribute(span, f"{trace_prefix}.{field}", value) + identity: Final = ( + *( + ((LangfuseSpanAttributes.OBSERVATION_METADATA.value, json.dumps(dict(observation_metadata))),) + if observation_metadata + else () + ), + *( + (f"{trace_prefix}.{field}", value) + for field in TRACE_IDENTITY_FIELDS + if (value := observation_metadata.get(field)) + ), + ) + limit: Final = span_attribute_limit(span) + carried: Final = carried_keys(span) + room: Final = ( + len(identity) if limit is None else max(limit - len(carried) - len(_ERROR_ATTRIBUTES - carried), 0) + ) + for key, value in identity[:room]: + _utils.safe_set_attribute(span, key, value) @staticmethod def _set_observation_output(span: Span, response_obj): diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 73448221acb..697b524f8f0 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -74,6 +74,7 @@ from litellm.integrations.otel.plumbing.metrics import ( create_genai_metrics, ) from litellm.integrations.otel.plumbing.providers import ( + LITELLM_TRACER_NAME, attach_tenant_fan_out, build_tracer_provider, get_event_logger, @@ -97,7 +98,6 @@ if TYPE_CHECKING: StandardLoggingPayload, ) -LITELLM_TRACER_NAME: Final = "litellm" _published_v2_provider: ApiTracerProvider | None = None diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 0f298d572b0..db56da033a5 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -374,6 +374,7 @@ class _DrainPool: self._backlog -= 1 +LITELLM_TRACER_NAME: Final = "litellm" _NO_ATTRIBUTES: Final[Mapping[str, AttributeValue]] = MappingProxyType({}) _DB_SYSTEM_KEYS: Final = frozenset({DB.SYSTEM_NAME, DB.SYSTEM_LEGACY}) # Keys on a database span that describe the proxy's own datastore: its host, its @@ -460,15 +461,46 @@ def without_message_content(span: ReadableSpan) -> ReadableSpan: return _SpanView(span, span.resource, kept, span.events, span.status, parent=span.parent) +class _CountedSpanView(_SpanView): + """A ``_SpanView`` of ``inner`` with other attributes that reports the dropped counts ``inner`` reports.""" + + def __init__(self, inner: ReadableSpan, attributes: Attributes) -> None: + super().__init__(inner, inner.resource, attributes, inner.events, inner.status, parent=inner.parent) + self._dropped_counts: Final = (inner.dropped_attributes, inner.dropped_events, inner.dropped_links) + + @property + def dropped_attributes(self) -> int: + return self._dropped_counts[0] + + @property + def dropped_events(self) -> int: + return self._dropped_counts[1] + + @property + def dropped_links(self) -> int: + return self._dropped_counts[2] + + +def _is_litellm_span(span: ReadableSpan) -> bool: + """Whether LiteLLM's own tracer wrote ``span``, rather than application or instrumentation code.""" + scope: Final = span.instrumentation_scope + return scope is not None and scope.name == LITELLM_TRACER_NAME + + def without_langfuse_request_metadata(span: ReadableSpan) -> ReadableSpan: - """``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination.""" + """``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination. + + Only LiteLLM's own spans lose it: any other span keeps every key its own code set. + """ attributes: Final = span.attributes or _NO_ATTRIBUTES - if not any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes): + if not (_is_litellm_span(span) and any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes)): return span - kept: Final = MappingProxyType( - {key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES} + return _CountedSpanView( + span, + MappingProxyType( + {key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES} + ), ) - return _SpanView(span, span.resource, kept, span.events, span.status, parent=span.parent) def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: @@ -627,34 +659,34 @@ def _for_destination( and the query string of the request URL, which can hold the same key. A destination whose effective setting disables span content loses the prompt and response content too, and a destination for any backend but Langfuse loses the - request metadata the Langfuse mapper writes. The span itself stays, so the tenant - still gets the whole trace tree. + request metadata the Langfuse mapper writes on LiteLLM's own spans. The span itself + stays, so the tenant still gets the whole trace tree. """ extra: Final = destination.resource_attributes defaults: Final = destination.resource_defaults - attributes: Final = span.attributes or _NO_ATTRIBUTES + source: Final = ( + span if destination.callback_name == ExporterOwner.LANGFUSE_OTEL else without_langfuse_request_metadata(span) + ) + attributes: Final = source.attributes or _NO_ATTRIBUTES database: Final = _is_database_span(attributes) owned: Final = _is_tenant_owned_span(attributes) unreachable: Final = _guardrail_unreachable(attributes) redacted: Final = not destination.captures_content(default_capture) - langfuse: Final = destination.callback_name == ExporterOwner.LANGFUSE_OTEL kept: Final = MappingProxyType( { key: _without_query(key, value) for key, value in attributes.items() - if _tenant_visible(key, database, owned, unreachable) - and not (redacted and _is_message_content(key)) - and (langfuse or key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES) + if _tenant_visible(key, database, owned, unreachable) and not (redacted and _is_message_content(key)) } ) - recorded: Final = span.events + recorded: Final = source.events events: Final = tuple(_without_stack_trace(event) for event in recorded) if owned else () unchanged: Final = owned and _same_attributes(kept, attributes) and all(a is b for a, b in zip(events, recorded)) if not extra and not defaults and unchanged: - return span - resource: Final = _tenant_resource(span.resource, defaults, extra) - status: Final = span.status if owned else Status(span.status.status_code) - return _SpanView(span, resource, kept, events, status, parent=span.parent) + return source + resource: Final = _tenant_resource(source.resource, defaults, extra) + status: Final = source.status if owned else Status(source.status.status_code) + return _SpanView(source, resource, kept, events, status, parent=source.parent) class TenantFanOutSpanProcessor(SpanProcessor): @@ -1123,7 +1155,7 @@ class _OverriddenBackendFilter(SpanProcessor): class _LangfuseRequestMetadataFilter(SpanProcessor): - """Strip the Langfuse request metadata from every span before ``inner`` exports it.""" + """Strip the Langfuse request metadata from LiteLLM's own spans before ``inner`` exports them.""" def __init__(self, inner: SpanProcessor) -> None: self._inner: Final = inner @@ -1450,8 +1482,8 @@ def build_tracer_provider( ``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel`` alone; a collector or any other backend in the same config keeps the full tree. - In a config with a ``langfuse_otel`` exporter, every other exporter goes without - the request metadata the Langfuse mapper writes. + In a config that writes the Langfuse vocabulary, every exporter but ``langfuse_otel``'s + goes without the request metadata the Langfuse mapper writes on LiteLLM's own spans. ``strip_message_content`` removes prompt and response content before every exporter, for a routed provider whose spans can carry content another destination asked for. @@ -1467,7 +1499,9 @@ def build_tracer_provider( # ``config._normalize`` guarantees at least one spec (it folds the top-level # ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty). - langfuse_owned: Final = any(spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters) + writes_langfuse: Final = "langfuse" in config.mapper_names or any( + spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters + ) for spec in config.exporters: if spec.requires_headers and not spec.headers: continue @@ -1475,7 +1509,7 @@ def build_tracer_provider( spec, use_simple_processor, strip_message_content, - strip_langfuse_request_metadata=langfuse_owned and spec.owner is not ExporterOwner.LANGFUSE_OTEL, + strip_langfuse_request_metadata=writes_langfuse and spec.owner is not ExporterOwner.LANGFUSE_OTEL, ) owner = spec.owner.value if tenant_overrides and spec.owner is not None else None scope = _operator_scope(config, spec) @@ -1624,7 +1658,7 @@ def _attached_processors(provider: trace.TracerProvider) -> "tuple[SpanProcessor return tuple(getattr(multi, "_span_processors", ())) -def get_tracer(provider: TracerProvider, name: str = "litellm") -> Tracer: +def get_tracer(provider: TracerProvider, name: str = LITELLM_TRACER_NAME) -> Tracer: # Stamp the instrumentation scope with the LiteLLM package version so every # emitted span carries a deterministic ``scope.version`` (the standard OTel # location for the emitting library's version) for downstream consumers. diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index d43ff4779bb..ddd9bca229b 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -1451,7 +1451,7 @@ def _limit_sweep( return {size: record for size, record in observed.items() if record is not None} -def test_langfuse_otel_identity_is_evicted_before_any_other_attribute_of_a_full_span( +def test_langfuse_otel_identity_never_evicts_or_drops_another_attribute_of_a_full_span( gateway: Gateway, tmp_path: Path ) -> None: marker: Final = "lf-limit-" + uuid.uuid4().hex @@ -1488,17 +1488,20 @@ def test_langfuse_otel_identity_is_evicted_before_any_other_attribute_of_a_full_ identity_count: Final = len(_identity_attributes(reference)) assert reference_dropped == 0, reference_dropped assert identity_count == len(_chat_identity_fields(identity)) + 1, sorted(reference) + kept_by_size: Final[dict[int, tuple[int, int]]] = {} # mutable-ok: filled once per swept span below for tool_count, (attrs, dropped) in spans.items(): - kept = _identity_attributes(attrs) - assert len(kept) == identity_count - min(dropped, identity_count), (tool_count, dropped, sorted(kept)) - if dropped <= identity_count: + kept = len(_identity_attributes(attrs)) + kept_by_size[tool_count] = (kept, dropped) + if kept: + assert dropped == 0, (tool_count, kept, dropped) expected = {"openinference.span.kind", "metadata", "llm.model_name", "llm.request.type"} - assert expected <= set(attrs), (tool_count, dropped, sorted(expected - set(attrs))) + assert expected <= set(attrs), (tool_count, sorted(expected - set(attrs))) tools = {f"llm.tools.{n}.name" for n in range(tool_count)} - assert tools <= set(attrs), (tool_count, dropped, sorted(tools - set(attrs))) - assert [tool_count for tool_count, (_, dropped) in spans.items() if 0 < dropped <= identity_count], { - tool_count: dropped for tool_count, (_, dropped) in spans.items() - } + assert tools <= set(attrs), (tool_count, sorted(tools - set(attrs))) + assert [size for size, (kept, dropped) in kept_by_size.items() if kept < identity_count and not dropped], ( + kept_by_size + ) + assert [size for size, (_, dropped) in kept_by_size.items() if dropped], kept_by_size def test_otel_v2_langfuse_identity_sheds_before_any_message_attribute(gateway: Gateway, tmp_path: Path) -> None: @@ -1550,3 +1553,168 @@ def test_otel_v2_langfuse_identity_sheds_before_any_message_attribute(gateway: G size for size, (shed_identity, shed_messages) in shed_by_size.items() if shed_identity and not shed_messages ], shed_by_size assert [size for size, (_, shed_messages) in shed_by_size.items() if shed_messages], shed_by_size + + +_APP_SPAN_HOOK: Final = """ +from opentelemetry import trace + +from litellm.integrations.custom_logger import CustomLogger + + +class AppSpanHook(CustomLogger): + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + with trace.get_tracer("lit8283-app").start_as_current_span("app.audit") as span: + span.set_attributes( + { + "app.response_id": response_obj.id, + "langfuse.observation.metadata": '{"source": "app"}', + "langfuse.trace.metadata.user_api_key_alias": "app-set-alias", + } + ) + + +instance = AppSpanHook() +""" + + +def test_otel_v2_span_another_tracer_wrote_keeps_its_langfuse_metadata_keys_at_the_collector( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-v2-app-" + uuid.uuid4().hex + (tmp_path / "lit8283_app_span.py").write_text(_APP_SPAN_HOOK) + with ( + wire_server(lambda _request: Reply()) as generic, + _langfuse_rig( + gateway, + tmp_path, + {"callbacks": ["langfuse_otel", "otel", "lit8283_app_span.instance"]}, + marker, + overrides={"LITELLM_OTEL_V2": "1", "OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": generic.url}, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + + response_id: Final = response.json()["id"] + + def app(span: Mapping[str, object]) -> bool: + return span.get("app.response_id") == response_id + + def llm(span: Mapping[str, object]) -> bool: + return _response_id(span.get("gen_ai.response.id")) == _response_id(response_id) + + spans: Final = eventually( + _watched_spans(generic), lambda seen: any(map(app, seen)) and any(map(llm, seen)), seconds=25 + ) + app_span: Final = next(span for span in spans if app(span)) + assert app_span["langfuse.observation.metadata"] == '{"source": "app"}', app_span + assert app_span["langfuse.trace.metadata.user_api_key_alias"] == "app-set-alias", app_span + collector_llm: Final = next(span for span in spans if llm(span)) + assert not _identity_attributes(collector_llm), sorted(collector_llm) + + +def _chat_span_drops(body: bytes) -> tuple[tuple[int, int, int], ...]: + request: Final = ExportTraceServiceRequest.FromString(body) + return tuple( + (span.dropped_attributes_count, span.dropped_events_count, span.dropped_links_count) + for resource in request.resource_spans + for scope in resource.scope_spans + for span in scope.spans + if span.name.startswith("chat ") + ) + + +def test_otel_v2_collector_keeps_the_dropped_counts_of_a_span_without_langfuse_request_metadata( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-v2-drops-" + uuid.uuid4().hex + with ( + wire_server(lambda _request: Reply()) as generic, + _langfuse_rig( + gateway, + tmp_path, + {"callbacks": ["langfuse_otel", "otel"]}, + marker, + overrides={ + "LITELLM_OTEL_V2": "1", + "OTEL_EXPORTER": "otlp_http", + "OTEL_ENDPOINT": generic.url, + "OTEL_SPAN_EVENT_COUNT_LIMIT": "0", + }, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": f"{marker} scripted -fail- upstream"}], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code >= 500, response.text + + drops: Final[dict[str, list[tuple[int, int, int]]]] = {"langfuse": [], "generic": []} # mutable-ok: drained + + def observed() -> dict[str, list[tuple[int, int, int]]]: + for name, wire in (("langfuse", rig.collector), ("generic", generic)): + drops[name].extend(drop for batch in wire.drain() for drop in _chat_span_drops(batch.body)) + return drops + + settled: Final = eventually( + observed, lambda seen: bool(seen["langfuse"]) and len(seen["generic"]) == len(seen["langfuse"]), seconds=25 + ) + assert any(events for _, events, _ in settled["langfuse"]), settled + assert sorted(settled["generic"]) == sorted(settled["langfuse"]), settled + + +def test_otel_v2_langfuse_mapper_sends_no_langfuse_request_metadata_to_a_plain_collector( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "lf-v2-mapper-" + uuid.uuid4().hex + with ( + _langfuse_rig( + gateway, + tmp_path, + {"callbacks": ["otel"]}, + marker, + overrides={ + "LITELLM_OTEL_V2": "1", + "MAPPER_NAMES": "genai,langfuse", + "OTEL_EXPORTER": "otlp_http", + "OTEL_ENDPOINT": "__collector__", + }, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = _scenario_identity(scenario, rig.provider) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": identity["model"], + "messages": [{"role": "user", "content": marker}], + "user": identity["end_user"], + "cache": {"no-cache": True}, + }, + key=key, + ) + assert response.status_code == 200, response.text + + collector: Final = _v2_llm_span(rig.collector, response.json()["id"]) + assert collector["langfuse.trace.metadata.team_id"] == identity["team_id"], collector + assert not _identity_attributes(collector), sorted(collector) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 7b79dc034df..fdd1e0fc451 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -18,7 +18,7 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -from opentelemetry.trace import Status, StatusCode +from opentelemetry.trace import Link, SpanContext, Status, StatusCode import litellm from litellm.integrations.custom_logger import CustomLogger @@ -5075,11 +5075,12 @@ _LANGFUSE_REQUEST_METADATA_SPAN: Final = MappingProxyType( "langfuse.trace.metadata.user_api_key_alias": "prod-key", "langfuse.trace.metadata.user_api_key_user_id": "user-1", "langfuse.trace.metadata.team_id": "team-9", + "gen_ai.operation.name": "chat", "gen_ai.request.model": "gpt-4o", } ) _WITHOUT_LANGFUSE_REQUEST_METADATA: Final = MappingProxyType( - {"langfuse.trace.metadata.team_id": "team-9", "gen_ai.request.model": "gpt-4o"} + {"langfuse.trace.metadata.team_id": "team-9", "gen_ai.operation.name": "chat", "gen_ai.request.model": "gpt-4o"} ) @@ -5087,10 +5088,19 @@ class TestLangfuseRequestMetadataStaysWithLangfuse: """The request metadata the Langfuse mapper writes reaches a Langfuse destination and no other.""" @staticmethod - def _emit(provider: TracerProvider, destinations: tuple[OtelDestination, ...] = ()) -> None: + def _emit( + provider: TracerProvider, + destinations: tuple[OtelDestination, ...] = (), + tracer_name: str = "litellm", + overflow: bool = False, + ) -> None: def run() -> None: set_request_destinations(destinations) - with get_tracer(provider, "litellm").start_as_current_span("chat gpt-4o") as span: + links: Final = (Link(SpanContext(trace_id=1, span_id=1, is_remote=True)),) if overflow else () + with get_tracer(provider, tracer_name).start_as_current_span("chat gpt-4o", links=links) as span: + if overflow: + span.set_attribute("app.evicted", "first") + span.add_event("app.event") span.set_attributes(_LANGFUSE_REQUEST_METADATA_SPAN) in_fresh_context(run) @@ -5112,6 +5122,19 @@ class TestLangfuseRequestMetadataStaysWithLangfuse: ) return build_tracer_provider(config, use_simple_processor=True, tenant_overrides=True), exporters + @staticmethod + def _fan_out() -> tuple[TracerProvider, InMemorySpanExporter, InMemorySpanExporter]: + langfuse_exporter, arize_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda destination: SimpleSpanProcessor( + langfuse_exporter if destination.callback_name == "langfuse_otel" else arize_exporter + ) + ) + ) + return provider, langfuse_exporter, arize_exporter + def test_a_collector_beside_the_langfuse_exporter_gets_no_langfuse_request_metadata( self, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -5124,27 +5147,49 @@ class TestLangfuseRequestMetadataStaysWithLangfuse: ) assert dict(by_name(exporters[None])["chat gpt-4o"].attributes) == dict(_WITHOUT_LANGFUSE_REQUEST_METADATA) - def test_a_collector_with_no_langfuse_exporter_keeps_what_its_mappers_write( + def test_a_collector_with_the_langfuse_mapper_and_no_langfuse_exporter_gets_no_langfuse_request_metadata( self, monkeypatch: pytest.MonkeyPatch ) -> None: provider, exporters = self._provider(monkeypatch, None) self._emit(provider) - assert dict(by_name(exporters[None])["chat gpt-4o"].attributes) == dict(_LANGFUSE_REQUEST_METADATA_SPAN) + assert dict(by_name(exporters[None])["chat gpt-4o"].attributes) == dict(_WITHOUT_LANGFUSE_REQUEST_METADATA) def test_the_fan_out_forwards_langfuse_request_metadata_to_langfuse_destinations_only(self) -> None: - langfuse_exporter, arize_exporter = InMemorySpanExporter(), InMemorySpanExporter() - provider: Final = TracerProvider() - provider.add_span_processor( - TenantFanOutSpanProcessor( - processor_factory=lambda destination: SimpleSpanProcessor( - langfuse_exporter if destination.callback_name == "langfuse_otel" else arize_exporter - ) - ) - ) + provider, langfuse_exporter, arize_exporter = self._fan_out() self._emit(provider, (LANGFUSE_DEST, INHERITING_DEST)) assert dict(by_name(langfuse_exporter)["chat gpt-4o"].attributes) == dict(_LANGFUSE_REQUEST_METADATA_SPAN) assert dict(by_name(arize_exporter)["chat gpt-4o"].attributes) == dict(_WITHOUT_LANGFUSE_REQUEST_METADATA) + + def test_a_span_another_tracer_wrote_keeps_the_langfuse_metadata_keys_it_set( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + provider, exporters = self._provider(monkeypatch, None, ExporterOwner.LANGFUSE_OTEL) + fanned, _langfuse_exporter, arize_exporter = self._fan_out() + + self._emit(provider, tracer_name="app") + self._emit(fanned, (LANGFUSE_DEST, INHERITING_DEST), tracer_name="app") + + assert dict(by_name(exporters[None])["chat gpt-4o"].attributes) == dict(_LANGFUSE_REQUEST_METADATA_SPAN) + assert dict(by_name(arize_exporter)["chat gpt-4o"].attributes) == dict(_LANGFUSE_REQUEST_METADATA_SPAN) + + def test_a_span_without_the_langfuse_request_metadata_keeps_its_dropped_counts( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", str(len(_LANGFUSE_REQUEST_METADATA_SPAN))) + monkeypatch.setenv("OTEL_SPAN_EVENT_COUNT_LIMIT", "0") + monkeypatch.setenv("OTEL_SPAN_LINK_COUNT_LIMIT", "0") + provider, exporters = self._provider(monkeypatch, None, ExporterOwner.LANGFUSE_OTEL) + fanned, _langfuse_exporter, arize_exporter = self._fan_out() + + self._emit(provider, overflow=True) + self._emit(fanned, (LANGFUSE_DEST, INHERITING_DEST), overflow=True) + + stripped: Final = (by_name(exporters[None])["chat gpt-4o"], by_name(arize_exporter)["chat gpt-4o"]) + assert [dict(span.attributes or {}) for span in stripped] == [dict(_WITHOUT_LANGFUSE_REQUEST_METADATA)] * 2 + assert [(span.dropped_attributes, span.dropped_events, span.dropped_links) for span in stripped] == [ + (1, 1, 1) + ] * 2 diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index eb7630781e4..fec14efa477 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -1,7 +1,9 @@ import base64 import json import os +from collections.abc import Mapping from datetime import datetime, timezone +from types import MappingProxyType from typing import Final from unittest.mock import MagicMock, patch @@ -700,9 +702,24 @@ class TestLangfuseOtelIntegration: assert json.loads(actual["langfuse.observation.metadata"]) == {"user_api_key_alias": "prod-key"} assert json.loads(actual["langfuse.observation.output"]) == {"role": "assistant", "content": "hello"} + # What ``OpenTelemetry._record_exception_on_span`` adds to a failed call's span after its attributes. + _FAILED_CALL_ATTRIBUTES: Final = MappingProxyType( + { + "error.code": "429", + "http.response.status_code": 429, + "error.type": "RateLimitError", + "error.message": "rate limited", + "error.llm_provider": "openai", + "error.stack_trace": "Traceback", + } + ) + @staticmethod def _exported_span( - kwargs: dict[str, object], max_span_attributes: int | None, carried: dict[str, str] | None = None + kwargs: dict[str, object], + max_span_attributes: int | None, + carried: dict[str, str] | None = None, + added_after: Mapping[str, str | int] = MappingProxyType({}), ) -> ReadableSpan: exporter: Final = InMemorySpanExporter() provider: Final = TracerProvider(span_limits=SpanLimits(max_span_attributes=max_span_attributes)) @@ -710,6 +727,7 @@ class TestLangfuseOtelIntegration: span: Final = provider.get_tracer("langfuse-otel-test").start_span("litellm_request", attributes=carried) response_obj: Final = {"id": "chatcmpl-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]} LangfuseOtelLogger.set_langfuse_otel_attributes(span, kwargs, response_obj) + span.set_attributes(added_after) span.end() (finished,) = exporter.get_finished_spans() return finished @@ -739,14 +757,16 @@ class TestLangfuseOtelIntegration: }, } - def test_request_metadata_never_evicts_another_attribute_at_the_span_limit( + def test_request_metadata_never_evicts_or_drops_another_attribute_at_the_span_limit( self, monkeypatch: pytest.MonkeyPatch ) -> None: kwargs: Final = self._tool_call_kwargs() + failed: Final = self._FAILED_CALL_ATTRIBUTES monkeypatch.setattr(litellm, "redact_user_api_key_info", True) - plain: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET).attributes or {}) + plain: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET, added_after=failed).attributes or {}) + succeeded: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET).attributes or {}) monkeypatch.setattr(litellm, "redact_user_api_key_info", False) - full: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET).attributes or {}) + full: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET, added_after=failed).attributes or {}) identity_keys: Final = set(full) - set(plain) assert identity_keys == { "langfuse.observation.metadata", @@ -754,12 +774,17 @@ class TestLangfuseOtelIntegration: "langfuse.trace.metadata.user_api_key_team_id", } - at_limit: Final = self._exported_span(kwargs, len(plain)) - assert dict(at_limit.attributes or {}) == plain - - one_spare: Final = dict(self._exported_span(kwargs, len(plain) + 1).attributes or {}) - assert {key: one_spare.get(key) for key in plain} == plain - assert len(set(one_spare) & identity_keys) == 1 + kept_identity: list[int] = [] # mutable-ok: one count per limit + for limit in range(len(plain), len(full) + 1): + failed_span = self._exported_span(kwargs, limit, added_after=failed) + succeeded_span = self._exported_span(kwargs, limit - len(failed)) + for span, expected in ((failed_span, plain), (succeeded_span, succeeded)): + attributes = dict(span.attributes or {}) + assert span.dropped_attributes == 0 + assert {key: attributes.get(key) for key in expected} == expected + assert set(attributes) - set(expected) <= identity_keys + kept_identity.append(len(set(failed_span.attributes or {}) & identity_keys)) + assert kept_identity == list(range(len(identity_keys) + 1)) def test_request_metadata_skips_a_span_that_already_carries_attributes(self) -> None: attributes: Final = dict( From e3b55c2c8e1b8f285b33b0344041e901fd8795b1 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 19 Oct 2026 23:33:41 -0700 Subject: [PATCH 26/28] fix(langfuse_otel): keep the caller identity off the caller's active span v1 writes LiteLLM's attributes onto the caller's active span when one is current instead of starting its own span. That span goes to the caller's own exporter, so the identity keys reached a non-Langfuse collector and could take attribute slots the caller fills later. The identity keys now go only on the span LiteLLM starts, whatever the caller's span carries. --- .../integrations/langfuse/langfuse_otel.py | 8 +- .../sdk/test_langfuse_otel_caller_span_sdk.py | 113 ++++++++++++++++++ tests/unit/integrations/test_langfuse_otel.py | 49 +++++++- 3 files changed, 161 insertions(+), 9 deletions(-) create mode 100644 tests/integration/sdk/test_langfuse_otel_caller_span_sdk.py diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 6db26c09c32..1444e735fed 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -63,10 +63,10 @@ class LangfuseOtelLogger(OpenTelemetry): Sets OpenTelemetry span attributes for Langfuse observability. Uses the same attribute setting logic as Arize Phoenix for consistency. """ - from opentelemetry.sdk.trace import ReadableSpan + from opentelemetry import trace - # A span that already carries attributes (a caller's active span) does not get the caller identity. - fresh: Final = not (isinstance(span, ReadableSpan) and span.attributes) + # The caller's active span, which LiteLLM writes onto instead of starting its own, gets no caller identity. + own_span: Final = span is not trace.get_current_span() _utils.set_attributes(span, kwargs, response_obj, LangfuseLLMObsOTELAttributes) span.set_attribute("langfuse.observation.type", "generation") @@ -74,7 +74,7 @@ class LangfuseOtelLogger(OpenTelemetry): # Set Langfuse specific attributes ######################################################### LangfuseOtelLogger._set_langfuse_specific_attributes(span=span, kwargs=kwargs, response_obj=response_obj) - if fresh: + if own_span: LangfuseOtelLogger._set_request_metadata_attributes( span=span, kwargs=_MAPPING_ADAPTER.validate_python(kwargs) ) diff --git a/tests/integration/sdk/test_langfuse_otel_caller_span_sdk.py b/tests/integration/sdk/test_langfuse_otel_caller_span_sdk.py new file mode 100644 index 00000000000..2e0f59974bd --- /dev/null +++ b/tests/integration/sdk/test_langfuse_otel_caller_span_sdk.py @@ -0,0 +1,113 @@ +import asyncio +import json +import time +from collections.abc import Mapping +from typing import Final + +import pytest +from integration._support.wire import Reply, Request, Wire, wire_server +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + +_IDENTITY: Final = { + "user_api_key_alias": "caller-span-alias", + "user_api_key_user_id": "caller-span-user", + "user_api_key_team_id": "caller-span-team", +} +_CHAT_REPLY: Final = Reply( + body=json.dumps( + { + "id": "chatcmpl-caller-span", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode() +) + + +def _is_identity(key: str) -> bool: + return key == "langfuse.observation.metadata" or key.startswith("langfuse.trace.metadata.user_api_key_") + + +def _exported(langfuse: Wire, seen: list[tuple[str, dict[str, object]]]) -> list[tuple[str, dict[str, object]]]: + for batch in langfuse.drain(): + request: ExportTraceServiceRequest = ExportTraceServiceRequest.FromString(batch.body) + seen.extend( + ( + span.name, + { + attribute.key: getattr(attribute.value, attribute.value.WhichOneof("value")) + for attribute in span.attributes + }, + ) + for resource in request.resource_spans + for scope in resource.scope_spans + for span in scope.spans + ) + return seen + + +async def _exported_until( + langfuse: Wire, seen: list[tuple[str, dict[str, object]]], name: str, prompt: str +) -> list[tuple[str, dict[str, object]]]: + """The spans the Langfuse sink got, once a ``name`` span for ``prompt`` is among them; polls without blocking the loop.""" + deadline: Final = time.monotonic() + 25 + while not any(span == name and prompt in json.dumps(attributes) for span, attributes in _exported(langfuse, seen)): + assert time.monotonic() < deadline, [span for span, _ in seen] + await asyncio.sleep(0.2) + return seen + + +async def _complete(upstream: Wire, prompt: str) -> None: + await litellm.acompletion( + model="openai/gpt-4o-mini", + api_base=upstream.url + "/v1", + api_key="synthetic-caller-span-key", + messages=[{"role": "user", "content": prompt}], + metadata=dict(_IDENTITY), + ) + await GLOBAL_LOGGING_WORKER.flush() + + +@pytest.mark.asyncio +async def test_langfuse_otel_keeps_the_caller_identity_off_the_callers_active_span( + monkeypatch: pytest.MonkeyPatch, +) -> None: + app_exporter: Final = InMemorySpanExporter() + app_provider: Final = TracerProvider() + app_provider.add_span_processor(SimpleSpanProcessor(app_exporter)) + + def chat(_request: Request) -> Reply: + return _CHAT_REPLY + + with wire_server(chat) as upstream, wire_server(lambda _request: Reply()) as langfuse: + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-caller-span") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-caller-span") + monkeypatch.setenv("LANGFUSE_OTEL_HOST", langfuse.url) + monkeypatch.setenv("OTEL_BSP_SCHEDULE_DELAY", "100") + monkeypatch.setattr(litellm, "callbacks", ["langfuse_otel"]) + + seen: Final[list[tuple[str, dict[str, object]]]] = [] # mutable-ok: drain() consumes the sink queue + with app_provider.get_tracer("app").start_as_current_span("app.operation"): + await _complete(upstream, "inside the caller span") + await _exported_until(langfuse, seen, "raw_gen_ai_request", "inside the caller span") + await _complete(upstream, "no caller span") + exported: Final = await _exported_until(langfuse, seen, "litellm_request", "no caller span") + + (app_span,) = app_exporter.get_finished_spans() + app_attributes: Final[Mapping[str, object]] = app_span.attributes or {} + assert app_attributes["langfuse.observation.type"] == "generation", sorted(app_attributes) + assert not [key for key in app_attributes if _is_identity(key)], sorted(app_attributes) + (own_span,) = [attributes for name, attributes in exported if name == "litellm_request"] + assert json.loads(str(own_span["langfuse.observation.metadata"])) == _IDENTITY, own_span + assert {key: value for key, value in own_span.items() if key.startswith("langfuse.trace.metadata.")} == { + f"langfuse.trace.metadata.{field}": value for field, value in _IDENTITY.items() + }, sorted(own_span) diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index fec14efa477..5b72f7e191e 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -2,12 +2,14 @@ import base64 import json import os from collections.abc import Mapping +from contextlib import nullcontext from datetime import datetime, timezone from types import MappingProxyType from typing import Final from unittest.mock import MagicMock, patch import pytest +from opentelemetry import trace from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -720,13 +722,15 @@ class TestLangfuseOtelIntegration: max_span_attributes: int | None, carried: dict[str, str] | None = None, added_after: Mapping[str, str | int] = MappingProxyType({}), + active: bool = False, ) -> ReadableSpan: exporter: Final = InMemorySpanExporter() provider: Final = TracerProvider(span_limits=SpanLimits(max_span_attributes=max_span_attributes)) provider.add_span_processor(SimpleSpanProcessor(exporter)) span: Final = provider.get_tracer("langfuse-otel-test").start_span("litellm_request", attributes=carried) response_obj: Final = {"id": "chatcmpl-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]} - LangfuseOtelLogger.set_langfuse_otel_attributes(span, kwargs, response_obj) + with trace.use_span(span, end_on_exit=False) if active else nullcontext(): + LangfuseOtelLogger.set_langfuse_otel_attributes(span, kwargs, response_obj) span.set_attributes(added_after) span.end() (finished,) = exporter.get_finished_spans() @@ -786,13 +790,13 @@ class TestLangfuseOtelIntegration: kept_identity.append(len(set(failed_span.attributes or {}) & identity_keys)) assert kept_identity == list(range(len(identity_keys) + 1)) - def test_request_metadata_skips_a_span_that_already_carries_attributes(self) -> None: + @pytest.mark.parametrize("carried", [None, {"caller.attribute": "kept"}], ids=["bare", "with_attributes"]) + def test_request_metadata_skips_the_callers_active_span(self, carried: dict[str, str] | None) -> None: attributes: Final = dict( - self._exported_span(self._tool_call_kwargs(), SpanLimits.UNSET, {"caller.attribute": "kept"}).attributes - or {} + self._exported_span(self._tool_call_kwargs(), SpanLimits.UNSET, carried, active=True).attributes or {} ) - assert attributes["caller.attribute"] == "kept" + assert {key: attributes[key] for key in carried or {}} == (carried or {}) assert attributes["langfuse.observation.type"] == "generation" assert "langfuse.observation.metadata" not in attributes assert not [key for key in attributes if key.startswith("langfuse.trace.metadata.")] @@ -1376,3 +1380,38 @@ def test_langfuse_otel_does_not_start_proxy_request_span() -> None: finally: langfuse_provider.shutdown() generic_provider.shutdown() + + +def test_langfuse_otel_success_writes_identity_onto_its_own_span_and_never_onto_the_callers() -> None: + langfuse_exporter: Final = InMemorySpanExporter() + langfuse_provider: Final = TracerProvider() + langfuse_provider.add_span_processor(SimpleSpanProcessor(langfuse_exporter)) + app_exporter: Final = InMemorySpanExporter() + app_provider: Final = TracerProvider() + app_provider.add_span_processor(SimpleSpanProcessor(app_exporter)) + with patch.dict(os.environ, LANGFUSE_ENV_ONLY, clear=True): + logger: Final = LangfuseOtelLogger(callback_name="langfuse_otel", tracer_provider=langfuse_provider) + response_obj: Final = {"id": "chatcmpl-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]} + started_at: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + try: + with app_provider.get_tracer("app").start_as_current_span("app.operation"): + logger._handle_success( + TestLangfuseOtelIntegration._tool_call_kwargs(), response_obj, started_at, started_at + ) + logger._handle_success(TestLangfuseOtelIntegration._tool_call_kwargs(), response_obj, started_at, started_at) + + (app_span,) = app_exporter.get_finished_spans() + (own_span,) = [span for span in langfuse_exporter.get_finished_spans() if span.name == "litellm_request"] + app_attributes: Final = dict(app_span.attributes or {}) + own_attributes: Final = dict(own_span.attributes or {}) + assert app_attributes["langfuse.observation.type"] == "generation" + identity_prefixes: Final = ("langfuse.observation.metadata", "langfuse.trace.metadata.") + assert not [key for key in app_attributes if key.startswith(identity_prefixes)] + assert json.loads(str(own_attributes["langfuse.observation.metadata"])) == { + "user_api_key_alias": "prod-key", + "user_api_key_team_id": "team-1", + } + assert own_attributes["langfuse.trace.metadata.user_api_key_team_id"] == "team-1" + finally: + langfuse_provider.shutdown() + app_provider.shutdown() From fe120be28d57f858d19db3d1f3af86a1fbc3b1ac Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 20 Oct 2026 17:36:24 -0700 Subject: [PATCH 27/28] fix(langfuse otel): keep the identity keys on the OTEL v2 Langfuse mapper only The v1 langfuse_otel callback goes back to main: it sends no langfuse.observation.metadata or langfuse.trace.metadata.user_api_key_* keys. The OTEL v2 Langfuse mapper keeps them. The integration cells now run every endpoint, client, redaction, failure and chaos case on v2. --- .../integrations/langfuse/langfuse_otel.py | 67 +--- litellm/types/integrations/langfuse_otel.py | 1 - .../test_langfuse_otel_metadata.py | 334 +++-------------- .../sdk/test_langfuse_otel_caller_span_sdk.py | 113 ------ tests/unit/integrations/test_langfuse_otel.py | 351 +----------------- 5 files changed, 47 insertions(+), 819 deletions(-) delete mode 100644 tests/integration/sdk/test_langfuse_otel_caller_span_sdk.py diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index fa978c04af2..f195cd97e3c 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -5,18 +5,13 @@ from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Optional -from pydantic import TypeAdapter - from litellm._logging import verbose_logger -from litellm.integrations._types.open_inference import ErrorAttributes from litellm.integrations.arize import _utils from litellm.integrations.langfuse.langfuse_otel_attributes import ( LangfuseLLMObsOTELAttributes, serialize_langfuse_datetime, ) -from litellm.integrations.opentelemetry import HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, OpenTelemetry, OpenTelemetryConfig -from litellm.integrations.otel.mappers.langfuse import TRACE_IDENTITY_FIELDS -from litellm.integrations.otel.model.metadata import IDENTITY_METADATA_FIELDS, exported_request_metadata +from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, @@ -31,19 +26,6 @@ else: Span = Any -_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) -# The attributes ``OpenTelemetry._record_exception_on_span`` adds to a failed call's span after these. -_ERROR_ATTRIBUTES: Final = frozenset( - { - ErrorAttributes.ERROR_CODE, - HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, - ErrorAttributes.ERROR_TYPE, - ErrorAttributes.ERROR_MESSAGE, - ErrorAttributes.ERROR_LLM_PROVIDER, - ErrorAttributes.ERROR_STACK_TRACE, - } -) - LANGFUSE_CLOUD_EU_ENDPOINT: Final = "https://cloud.langfuse.com/api/public/otel" LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/otel" LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version" @@ -64,10 +46,7 @@ class LangfuseOtelLogger(OpenTelemetry): Sets OpenTelemetry span attributes for Langfuse observability. Uses the same attribute setting logic as Arize Phoenix for consistency. """ - from opentelemetry import trace - # The caller's active span, which LiteLLM writes onto instead of starting its own, gets no caller identity. - own_span: Final = span is not trace.get_current_span() _utils.set_attributes(span, kwargs, response_obj, LangfuseLLMObsOTELAttributes) span.set_attribute("langfuse.observation.type", "generation") @@ -75,10 +54,6 @@ class LangfuseOtelLogger(OpenTelemetry): # Set Langfuse specific attributes ######################################################### LangfuseOtelLogger._set_langfuse_specific_attributes(span=span, kwargs=kwargs, response_obj=response_obj) - if own_span: - LangfuseOtelLogger._set_request_metadata_attributes( - span=span, kwargs=_MAPPING_ADAPTER.validate_python(kwargs) - ) @staticmethod def _extract_langfuse_metadata(kwargs: Mapping[str, Any]) -> dict: @@ -152,46 +127,6 @@ class LangfuseOtelLogger(OpenTelemetry): value = str(value) safe_set_attribute(span, enum_attr.value, value) - @staticmethod - def _set_request_metadata_attributes( - span: Span, - kwargs: Mapping[object, object], - ) -> None: - """Set the caller identity into the room ``span``'s attribute limit leaves next to the error attributes - a failed call adds after it, so the identity never evicts or drops another attribute.""" - from litellm.integrations.otel.emitter import carried_keys, span_attribute_limit - - raw_standard_logging_object: Final = kwargs.get("standard_logging_object") - if not isinstance(raw_standard_logging_object, dict): - return - raw_metadata: Final = _MAPPING_ADAPTER.validate_python(raw_standard_logging_object).get("metadata") - if not isinstance(raw_metadata, dict): - return - metadata: Final = _MAPPING_ADAPTER.validate_python(raw_metadata) - observation_metadata: Final = exported_request_metadata( - {field: metadata.get(field) for field in IDENTITY_METADATA_FIELDS} - ) - trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value - identity: Final = ( - *( - ((LangfuseSpanAttributes.OBSERVATION_METADATA.value, json.dumps(dict(observation_metadata))),) - if observation_metadata - else () - ), - *( - (f"{trace_prefix}.{field}", value) - for field in TRACE_IDENTITY_FIELDS - if (value := observation_metadata.get(field)) - ), - ) - limit: Final = span_attribute_limit(span) - carried: Final = carried_keys(span) - room: Final = ( - len(identity) if limit is None else max(limit - len(carried) - len(_ERROR_ATTRIBUTES - carried), 0) - ) - for key, value in identity[:room]: - _utils.safe_set_attribute(span, key, value) - @staticmethod def _set_observation_output(span: Span, response_obj): """Helper to set observation output attributes.""" diff --git a/litellm/types/integrations/langfuse_otel.py b/litellm/types/integrations/langfuse_otel.py index 33b708b2ac6..5647f28c757 100644 --- a/litellm/types/integrations/langfuse_otel.py +++ b/litellm/types/integrations/langfuse_otel.py @@ -30,7 +30,6 @@ class LangfuseSpanAttributes(str, Enum): OBSERVATION_INPUT = "langfuse.observation.input" OBSERVATION_OUTPUT = "langfuse.observation.output" OBSERVATION_COMPLETION_START_TIME = "langfuse.observation.completion_start_time" - OBSERVATION_METADATA = "langfuse.observation.metadata" # ---- Trace-level metadata ---- TRACE_USER_ID = "user.id" diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index ddd9bca229b..78b31d8ee4e 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -256,6 +256,7 @@ def _langfuse_rig( "LANGFUSE_PUBLIC_KEY": "pk-lf-integration", "LANGFUSE_SECRET_KEY": "sk-lf-integration", "LANGFUSE_HOST": collector.url, + "LITELLM_OTEL_V2": "1", **{ name: (collector.url if value == "__collector__" else value) for name, value in dict(overrides or {}).items() @@ -282,7 +283,7 @@ def _response_id(value: object) -> str | None: def _span_response_id(span: Mapping[str, object]) -> str | None: - return _response_id(span.get("llm.response.id")) + return _response_id(span.get("gen_ai.response.id")) def _watched_spans(collector: Wire) -> Callable[[], tuple[dict[str, object], ...]]: @@ -307,32 +308,6 @@ def _v2_llm_span(collector: Wire, response_id: str) -> dict[str, object]: return next(span for span in observed if matches(span)) -def _observation_span(collector: Wire, response_id: str) -> dict[str, object]: - def spans() -> tuple[dict[str, object], ...]: - return tuple( - attributes - for batch in collector.drain() - for attributes in _span_attributes(batch.body) - if _span_response_id(attributes) == _response_id(response_id) - ) - - observed: Final = eventually(spans, lambda values: len(values) == 1, seconds=25) - return observed[0] - - -def _marked_span(collector: Wire, marker: str, needle: str) -> dict[str, object]: - def spans() -> tuple[dict[str, object], ...]: - return tuple( - attributes - for batch in collector.drain() - for attributes in _span_attributes(batch.body) - if needle in json.dumps(attributes, default=str) and marker in json.dumps(attributes, default=str) - ) - - observed: Final = eventually(spans, lambda values: len(values) == 1, seconds=25) - return observed[0] - - def _scenario_identity( scenario: Scenario, provider: Wire, @@ -360,7 +335,6 @@ def _scenario_identity( def _assert_identity(attrs: Mapping[str, object], expected: Mapping[str, object]) -> None: - assert "metadata" in attrs, sorted(attrs) assert "langfuse.observation.metadata" in attrs, sorted(attrs) observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) assert {key: observation.get(key) for key in expected} == dict(expected), observation @@ -383,63 +357,6 @@ def _chat_identity_fields(identity: Mapping[str, str | None]) -> dict[str, objec } -def test_langfuse_otel_emits_request_metadata_under_langfuse_observation_and_trace_keys( - gateway: Gateway, tmp_path: Path -) -> None: - marker: Final = "lf-meta-" + uuid.uuid4().hex - with ( - _langfuse_rig(gateway, tmp_path, {}, marker) as rig, - rig.candidate.scenario() as scenario, - ): - key, identity = _scenario_identity(scenario, rig.provider) - response: Final = rig.candidate.request( - "POST", - "/v1/chat/completions", - { - "model": identity["model"], - "messages": [{"role": "user", "content": marker}], - "user": identity["end_user"], - "cache": {"no-cache": True}, - }, - key=key, - ) - assert response.status_code == 200, response.text - assert response.json()["choices"][0]["message"]["content"] == "langfuse-echo" - - attrs: Final = _observation_span(rig.collector, response.json()["id"]) - _assert_identity(attrs, _chat_identity_fields(identity)) - - -def test_langfuse_otel_metadata_redacts_user_api_key_fields_like_vanilla_langfuse( - gateway: Gateway, tmp_path: Path -) -> None: - marker: Final = "lf-redact-" + uuid.uuid4().hex - with ( - _langfuse_rig(gateway, tmp_path, {"redact_user_api_key_info": True}, marker) as rig, - rig.candidate.scenario() as scenario, - ): - key: Final = scenario.key( - key_alias="lf-redact-" + uuid.uuid4().hex, - metadata={"spend_logs_metadata": {"ticket": "LIT-8283"}}, - ) - model: Final = scenario.model(api_base=rig.provider.url + "/v1") - response: Final = rig.candidate.request( - "POST", - "/v1/chat/completions", - { - "model": model, - "messages": [{"role": "user", "content": marker}], - "cache": {"no-cache": True}, - }, - key=key, - ) - assert response.status_code == 200, response.text - - attrs: Final = _observation_span(rig.collector, response.json()["id"]) - assert "langfuse.observation.metadata" not in attrs, attrs["langfuse.observation.metadata"] - assert not [name for name in attrs if name.startswith("langfuse.trace.metadata.")], sorted(attrs) - - def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-sdkstream-" + uuid.uuid4().hex with ( @@ -463,7 +380,7 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_stream(gateway: ) assert chunks, "stream produced no chunks" response_id: Final = chunks[-1].id - attrs: Final = _observation_span(rig.collector, response_id) + attrs: Final = _v2_llm_span(rig.collector, response_id) _assert_identity(attrs, _chat_identity_fields(identity)) @@ -485,7 +402,7 @@ async def test_langfuse_otel_metadata_reaches_langfuse_for_openai_sdk_async(gate user=identity["end_user"], extra_body={"cache": {"no-cache": True}}, ) - attrs: Final = _observation_span(rig.collector, response.id) + attrs: Final = _v2_llm_span(rig.collector, response.id) _assert_identity(attrs, _chat_identity_fields(identity)) @@ -508,7 +425,7 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_messages(gate messages=[{"role": "user", "content": marker}], extra_body={"litellm_metadata": {"tags": ["lf8283"]}, "metadata": {"user_id": identity["end_user"]}}, ) - attrs: Final = _observation_span(rig.collector, message.id) + attrs: Final = _v2_llm_span(rig.collector, message.id) _assert_identity(attrs, _chat_identity_fields(identity)) @@ -536,7 +453,7 @@ async def test_langfuse_otel_metadata_reaches_langfuse_for_anthropic_sdk_stream( ) as stream, ): final: Final = await stream.get_final_message() - attrs: Final = _observation_span(rig.collector, final.id) + attrs: Final = _v2_llm_span(rig.collector, final.id) _assert_identity(attrs, _chat_identity_fields(identity)) @@ -558,7 +475,7 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_api(gateway: Gate user=identity["end_user"], extra_body={"cache": {"no-cache": True}}, ) - attrs: Final = _observation_span(rig.collector, response.id) + attrs: Final = _v2_llm_span(rig.collector, response.id) _assert_identity(attrs, _chat_identity_fields(identity)) @@ -589,32 +506,7 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gatewa response_id: Final = next( event["response_id"] for event in events if event.get("type") == "response.output_text.delta" ) - attrs: Final = _observation_span(rig.collector, response_id) - _assert_identity(attrs, _chat_identity_fields(identity)) - - -def test_langfuse_otel_caller_trace_metadata_survives_flattened_identity(gateway: Gateway, tmp_path: Path) -> None: - marker: Final = "lf-deploy-" + uuid.uuid4().hex - with ( - _langfuse_rig(gateway, tmp_path, {}, marker) as rig, - rig.candidate.scenario() as scenario, - ): - key, identity = _scenario_identity(scenario, rig.provider) - response: Final = rig.candidate.request( - "POST", - "/v1/chat/completions", - { - "model": identity["model"], - "messages": [{"role": "user", "content": marker}], - "user": identity["end_user"], - "metadata": {"trace_metadata": {"deploy": "blue"}, "tags": ["lf8283"]}, - "cache": {"no-cache": True}, - }, - key=key, - ) - assert response.status_code == 200, response.text - attrs: Final = _observation_span(rig.collector, response.json()["id"]) - assert json.loads(str(attrs["langfuse.trace.metadata"])) == {"deploy": "blue"}, attrs + attrs: Final = _v2_llm_span(rig.collector, response_id) _assert_identity(attrs, _chat_identity_fields(identity)) @@ -637,65 +529,19 @@ def test_langfuse_otel_failure_span_carries_request_metadata(gateway: Gateway, t key=key, ) assert response.status_code >= 500, response.text - attrs: Final = _marked_span(rig.collector, marker, "langfuse.observation.") - _assert_identity(attrs, _chat_identity_fields(identity)) + def failed_calls(spans: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple( + span + for span in spans + if span.get("langfuse.trace.metadata.team_id") == identity["team_id"] and "error.type" in span + ) -def test_langfuse_otel_teamless_key_emits_no_team_identity(gateway: Gateway, tmp_path: Path) -> None: - marker: Final = "lf-noteam-" + uuid.uuid4().hex - with ( - _langfuse_rig(gateway, tmp_path, {}, marker) as rig, - rig.candidate.scenario() as scenario, - ): - key, identity = _scenario_identity(scenario, rig.provider, team=False) - response: Final = rig.candidate.request( - "POST", - "/v1/chat/completions", - { - "model": identity["model"], - "messages": [{"role": "user", "content": marker}], - "cache": {"no-cache": True}, - }, - key=key, + failed: Final = failed_calls( + eventually(_watched_spans(rig.collector), lambda spans: bool(failed_calls(spans)), seconds=25) ) - assert response.status_code == 200, response.text - attrs: Final = _observation_span(rig.collector, response.json()["id"]) - assert "langfuse.observation.metadata" in attrs, sorted(attrs) - observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) - for field in ("user_api_key_team_id", "user_api_key_team_alias", "user_api_key_end_user_id"): - assert field not in observation, observation - assert set(observation) <= _EXPORTED_FIELDS, observation - assert { - attribute: attrs.get(attribute) - for attribute in (f"langfuse.trace.metadata.{field}" for field in _IDENTITIES) - if attribute in attrs - } == {"langfuse.trace.metadata.user_api_key_alias": identity["key_alias"]}, attrs - - -def test_langfuse_otel_teamless_key_keeps_caller_trace_metadata(gateway: Gateway, tmp_path: Path) -> None: - marker: Final = "lf-noteam-meta-" + uuid.uuid4().hex - with ( - _langfuse_rig(gateway, tmp_path, {}, marker) as rig, - rig.candidate.scenario() as scenario, - ): - key, identity = _scenario_identity(scenario, rig.provider, team=False) - response: Final = rig.candidate.request( - "POST", - "/v1/chat/completions", - { - "model": identity["model"], - "messages": [{"role": "user", "content": marker}], - "metadata": {"trace_metadata": {"deploy": "blue"}}, - "cache": {"no-cache": True}, - }, - key=key, - ) - assert response.status_code == 200, response.text - attrs: Final = _observation_span(rig.collector, response.json()["id"]) - assert json.loads(str(attrs["langfuse.trace.metadata"])) == {"deploy": "blue"}, attrs - assert "langfuse.trace.metadata.user_api_key_team_id" not in attrs, attrs - assert "langfuse.trace.metadata.user_api_key_team_alias" not in attrs, attrs - assert attrs.get("langfuse.trace.metadata.user_api_key_alias") == identity["key_alias"], attrs + for attrs in failed: + _assert_identity(attrs, _chat_identity_fields(identity)) def test_langfuse_otel_oversized_caller_metadata_is_not_copied_into_the_langfuse_keys( @@ -727,11 +573,18 @@ def test_langfuse_otel_oversized_caller_metadata_is_not_copied_into_the_langfuse key=key, ) assert response.status_code == 200, response.text - attrs: Final = _observation_span(rig.collector, response.json()["id"]) + attrs: Final = _v2_llm_span(rig.collector, response.json()["id"]) _assert_identity(attrs, _chat_identity_fields(identity)) assert len(str(attrs["langfuse.observation.metadata"])) < 1000, len(str(attrs["langfuse.observation.metadata"])) - assert caller_note in str(attrs["metadata"]), "the bare metadata attribute keeps the caller metadata" - assert spend_logs_metadata["blob"] in str(attrs["metadata"]), "the bare metadata attribute keeps it" + langfuse_metadata: Final = json.dumps( + { + name: value + for name, value in attrs.items() + if name == "langfuse.observation.metadata" or name.startswith("langfuse.trace.metadata.") + } + ) + assert caller_note not in langfuse_metadata, len(langfuse_metadata) + assert spend_logs_metadata["blob"] not in langfuse_metadata, len(langfuse_metadata) def test_langfuse_otel_unauthenticated_request_leaks_no_identity(gateway: Gateway, tmp_path: Path) -> None: @@ -779,20 +632,13 @@ def test_langfuse_otel_unauthenticated_request_leaks_no_identity(gateway: Gatewa settled: Final = eventually( watch, lambda spans: sum(_span_response_id(s) == second_id for s in spans) == 1, seconds=25 ) - denied_spans: Final = tuple(span for span in settled if f"{marker}-denied" in json.dumps(span, default=str)) - assert len(denied_spans) == 1, [s.get("llm.response.id") for s in settled] - denied_span: Final = denied_spans[0] - assert not [name for name in denied_span if name.startswith("langfuse.trace.metadata.user_api_key")], ( - denied_span + carriers: Final = tuple( + _span_response_id(span) + for span in settled + if "langfuse.observation.metadata" in span + or any(name.startswith("langfuse.trace.metadata.user_api_key_") for name in span) ) - leaked: Final = json.loads(str(denied_span.get("langfuse.observation.metadata", "{}"))) - for field in ( - "user_api_key_alias", - "user_api_key_team_id", - "user_api_key_team_alias", - "user_api_key_end_user_id", - ): - assert field not in leaked, leaked + assert sorted(carriers, key=str) == sorted((first_id, second_id)), carriers def test_langfuse_otel_repeated_identical_requests_emit_one_span_each(gateway: Gateway, tmp_path: Path) -> None: @@ -903,10 +749,12 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa watch, lambda spans: all(sum(_span_response_id(s) == rid for s in spans) == 1 for rid in response_ids), seconds=80, + return_last_on_timeout=True, ) - assert sorted(_span_response_id(span) for span in settled if _span_response_id(span) in response_ids) == sorted( - response_ids + landed: Final = sorted( + str(_span_response_id(span)) for span in settled if _span_response_id(span) in response_ids ) + assert landed == sorted(response_ids), sorted(set(response_ids) - set(landed)) assert len(rejected) >= 1, "sink outage never served a 503" @@ -1006,7 +854,6 @@ def test_langfuse_otel_v2_mapper_emits_request_metadata_and_identity(gateway: Ga tmp_path, {}, marker, - overrides={"LITELLM_OTEL_V2": "1"}, ) as rig, rig.candidate.scenario() as scenario, ): @@ -1053,7 +900,6 @@ def test_langfuse_otel_v2_mapper_redacts_user_api_key_fields(gateway: Gateway, t tmp_path, {"redact_user_api_key_info": True}, marker, - overrides={"LITELLM_OTEL_V2": "1"}, ) as rig, rig.candidate.scenario() as scenario, ): @@ -1092,7 +938,6 @@ def test_langfuse_otel_v2_mapper_teamless_key_emits_no_team_identity(gateway: Ga tmp_path, {}, marker, - overrides={"LITELLM_OTEL_V2": "1"}, ) as rig, rig.candidate.scenario() as scenario, ): @@ -1154,7 +999,6 @@ def test_otel_v2_scalar_requester_metadata_keeps_the_llm_span( {"callbacks": [callback]}, marker, overrides={ - "LITELLM_OTEL_V2": "1", **({"OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": "__collector__"} if callback == "otel" else {}), }, ) as rig, @@ -1179,26 +1023,6 @@ def test_otel_v2_scalar_requester_metadata_keeps_the_llm_span( assert observation["user_api_key_alias"] == identity["key_alias"], observation -def test_langfuse_otel_scalar_requester_metadata_keeps_the_observation_input(gateway: Gateway, tmp_path: Path) -> None: - marker: Final = "lf-scalar-" + uuid.uuid4().hex - with ( - _langfuse_rig(gateway, tmp_path, {}, marker) as rig, - rig.candidate.scenario() as scenario, - ): - key, identity = _scenario_identity(scenario, rig.provider) - model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) - response: Final = rig.candidate.request( - "POST", "/v1/messages", _scalar_requester_metadata_request("/v1/messages", model, marker), key=key - ) - assert response.status_code == 200, response.text - - attrs: Final = _observation_span(rig.collector, response.json()["id"]) - assert marker in str(attrs.get("langfuse.observation.input")), sorted(attrs) - observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) - assert "requester_metadata" not in observation, observation - assert observation["user_api_key_alias"] == identity["key_alias"], observation - - @pytest.mark.parametrize( "callbacks", [["langfuse_otel", "otel"], ["langfuse_otel"]], @@ -1216,7 +1040,7 @@ def test_otel_v2_collector_beside_langfuse_gets_no_langfuse_request_metadata( tmp_path, {"callbacks": callbacks}, marker, - overrides={"LITELLM_OTEL_V2": "1", "OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": generic.url}, + overrides={"OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": generic.url}, ) as rig, rig.candidate.scenario() as scenario, ): @@ -1258,7 +1082,7 @@ def test_otel_v2_langfuse_gets_no_request_headers_or_caller_metadata( marker: Final = "lf-v2-probe-" + uuid.uuid4().hex probe: Final = "lf-probe-" + uuid.uuid4().hex with ( - _langfuse_rig(gateway, tmp_path, {}, marker, overrides={"LITELLM_OTEL_V2": "1"}) as rig, + _langfuse_rig(gateway, tmp_path, {}, marker) as rig, rig.candidate.scenario() as scenario, ): key, identity = _scenario_identity(scenario, rig.provider) @@ -1346,10 +1170,7 @@ def _keyword_router(router: str, keyword: str) -> Callable[[str], Sequence[Mappi return models -@pytest.mark.parametrize("otel_v2", [False, True], ids=["v1", "v2"]) -def test_langfuse_otel_metadata_omits_prompt_quoting_routing_fields( - otel_v2: bool, gateway: Gateway, tmp_path: Path -) -> None: +def test_langfuse_otel_metadata_omits_prompt_quoting_routing_fields(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-route-" + uuid.uuid4().hex router: Final = "lf-router-" + uuid.uuid4().hex[:8] keyword: Final = "acquire-globex-" + uuid.uuid4().hex[:8] @@ -1359,11 +1180,7 @@ def test_langfuse_otel_metadata_omits_prompt_quoting_routing_fields( tmp_path, {}, marker, - overrides=( - {"LITELLM_OTEL_V2": "1", "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "no_content"} - if otel_v2 - else None - ), + overrides={"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "no_content"}, model_list=_keyword_router(router, keyword), ) as rig, rig.candidate.scenario() as scenario, @@ -1381,11 +1198,7 @@ def test_langfuse_otel_metadata_omits_prompt_quoting_routing_fields( ) assert response.status_code == 200, response.text - attrs: Final = ( - _v2_llm_span(rig.collector, response.json()["id"]) - if otel_v2 - else _observation_span(rig.collector, response.json()["id"]) - ) + attrs: Final = _v2_llm_span(rig.collector, response.json()["id"]) observation: Final = json.loads(str(attrs["langfuse.observation.metadata"])) assert set(observation) <= _EXPORTED_FIELDS, observation langfuse_metadata: Final = { @@ -1394,8 +1207,7 @@ def test_langfuse_otel_metadata_omits_prompt_quoting_routing_fields( if name == "langfuse.observation.metadata" or name.startswith("langfuse.trace.metadata.") } assert keyword not in json.dumps(langfuse_metadata, default=str), langfuse_metadata - if otel_v2: - assert keyword not in json.dumps(attrs, default=str), attrs + assert keyword not in json.dumps(attrs, default=str), attrs def _span_records(body: bytes) -> tuple[tuple[dict[str, object], int], ...]: @@ -1451,59 +1263,6 @@ def _limit_sweep( return {size: record for size, record in observed.items() if record is not None} -def test_langfuse_otel_identity_never_evicts_or_drops_another_attribute_of_a_full_span( - gateway: Gateway, tmp_path: Path -) -> None: - marker: Final = "lf-limit-" + uuid.uuid4().hex - with ( - _langfuse_rig(gateway, tmp_path, {}, marker) as rig, - rig.candidate.scenario() as scenario, - ): - key, identity = _scenario_identity(scenario, rig.provider) - - def body(tool_count: int) -> dict[str, object]: - return { - "model": identity["model"], - "messages": [{"role": "user", "content": f"{marker}:{tool_count}:"}], - "user": identity["end_user"], - "tools": [ - { - "type": "function", - "function": { - "name": f"tool_{n}", - "description": f"looks up thing {n}", - "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, - }, - } - for n in range(tool_count) - ], - "cache": {"no-cache": True}, - } - - spans: Final = _limit_sweep( - rig, key, marker, {tool_count: body(tool_count) for tool_count in (0, *range(30, 41))} - ) - - reference, reference_dropped = spans[0] - identity_count: Final = len(_identity_attributes(reference)) - assert reference_dropped == 0, reference_dropped - assert identity_count == len(_chat_identity_fields(identity)) + 1, sorted(reference) - kept_by_size: Final[dict[int, tuple[int, int]]] = {} # mutable-ok: filled once per swept span below - for tool_count, (attrs, dropped) in spans.items(): - kept = len(_identity_attributes(attrs)) - kept_by_size[tool_count] = (kept, dropped) - if kept: - assert dropped == 0, (tool_count, kept, dropped) - expected = {"openinference.span.kind", "metadata", "llm.model_name", "llm.request.type"} - assert expected <= set(attrs), (tool_count, sorted(expected - set(attrs))) - tools = {f"llm.tools.{n}.name" for n in range(tool_count)} - assert tools <= set(attrs), (tool_count, sorted(tools - set(attrs))) - assert [size for size, (kept, dropped) in kept_by_size.items() if kept < identity_count and not dropped], ( - kept_by_size - ) - assert [size for size, (_, dropped) in kept_by_size.items() if dropped], kept_by_size - - def test_otel_v2_langfuse_identity_sheds_before_any_message_attribute(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-v2-limit-" + uuid.uuid4().hex with ( @@ -1513,7 +1272,6 @@ def test_otel_v2_langfuse_identity_sheds_before_any_message_attribute(gateway: G {}, marker, overrides={ - "LITELLM_OTEL_V2": "1", "MAPPER_NAMES": "genai,openinference", "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "span_only", }, @@ -1589,7 +1347,7 @@ def test_otel_v2_span_another_tracer_wrote_keeps_its_langfuse_metadata_keys_at_t tmp_path, {"callbacks": ["langfuse_otel", "otel", "lit8283_app_span.instance"]}, marker, - overrides={"LITELLM_OTEL_V2": "1", "OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": generic.url}, + overrides={"OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": generic.url}, ) as rig, rig.candidate.scenario() as scenario, ): @@ -1647,7 +1405,6 @@ def test_otel_v2_collector_keeps_the_dropped_counts_of_a_span_without_langfuse_r {"callbacks": ["langfuse_otel", "otel"]}, marker, overrides={ - "LITELLM_OTEL_V2": "1", "OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": generic.url, "OTEL_SPAN_EVENT_COUNT_LIMIT": "0", @@ -1693,7 +1450,6 @@ def test_otel_v2_langfuse_mapper_sends_no_langfuse_request_metadata_to_a_plain_c {"callbacks": ["otel"]}, marker, overrides={ - "LITELLM_OTEL_V2": "1", "MAPPER_NAMES": "genai,langfuse", "OTEL_EXPORTER": "otlp_http", "OTEL_ENDPOINT": "__collector__", diff --git a/tests/integration/sdk/test_langfuse_otel_caller_span_sdk.py b/tests/integration/sdk/test_langfuse_otel_caller_span_sdk.py deleted file mode 100644 index 2e0f59974bd..00000000000 --- a/tests/integration/sdk/test_langfuse_otel_caller_span_sdk.py +++ /dev/null @@ -1,113 +0,0 @@ -import asyncio -import json -import time -from collections.abc import Mapping -from typing import Final - -import pytest -from integration._support.wire import Reply, Request, Wire, wire_server -from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - -import litellm -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER - -_IDENTITY: Final = { - "user_api_key_alias": "caller-span-alias", - "user_api_key_user_id": "caller-span-user", - "user_api_key_team_id": "caller-span-team", -} -_CHAT_REPLY: Final = Reply( - body=json.dumps( - { - "id": "chatcmpl-caller-span", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o-mini", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - ).encode() -) - - -def _is_identity(key: str) -> bool: - return key == "langfuse.observation.metadata" or key.startswith("langfuse.trace.metadata.user_api_key_") - - -def _exported(langfuse: Wire, seen: list[tuple[str, dict[str, object]]]) -> list[tuple[str, dict[str, object]]]: - for batch in langfuse.drain(): - request: ExportTraceServiceRequest = ExportTraceServiceRequest.FromString(batch.body) - seen.extend( - ( - span.name, - { - attribute.key: getattr(attribute.value, attribute.value.WhichOneof("value")) - for attribute in span.attributes - }, - ) - for resource in request.resource_spans - for scope in resource.scope_spans - for span in scope.spans - ) - return seen - - -async def _exported_until( - langfuse: Wire, seen: list[tuple[str, dict[str, object]]], name: str, prompt: str -) -> list[tuple[str, dict[str, object]]]: - """The spans the Langfuse sink got, once a ``name`` span for ``prompt`` is among them; polls without blocking the loop.""" - deadline: Final = time.monotonic() + 25 - while not any(span == name and prompt in json.dumps(attributes) for span, attributes in _exported(langfuse, seen)): - assert time.monotonic() < deadline, [span for span, _ in seen] - await asyncio.sleep(0.2) - return seen - - -async def _complete(upstream: Wire, prompt: str) -> None: - await litellm.acompletion( - model="openai/gpt-4o-mini", - api_base=upstream.url + "/v1", - api_key="synthetic-caller-span-key", - messages=[{"role": "user", "content": prompt}], - metadata=dict(_IDENTITY), - ) - await GLOBAL_LOGGING_WORKER.flush() - - -@pytest.mark.asyncio -async def test_langfuse_otel_keeps_the_caller_identity_off_the_callers_active_span( - monkeypatch: pytest.MonkeyPatch, -) -> None: - app_exporter: Final = InMemorySpanExporter() - app_provider: Final = TracerProvider() - app_provider.add_span_processor(SimpleSpanProcessor(app_exporter)) - - def chat(_request: Request) -> Reply: - return _CHAT_REPLY - - with wire_server(chat) as upstream, wire_server(lambda _request: Reply()) as langfuse: - monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-caller-span") - monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-caller-span") - monkeypatch.setenv("LANGFUSE_OTEL_HOST", langfuse.url) - monkeypatch.setenv("OTEL_BSP_SCHEDULE_DELAY", "100") - monkeypatch.setattr(litellm, "callbacks", ["langfuse_otel"]) - - seen: Final[list[tuple[str, dict[str, object]]]] = [] # mutable-ok: drain() consumes the sink queue - with app_provider.get_tracer("app").start_as_current_span("app.operation"): - await _complete(upstream, "inside the caller span") - await _exported_until(langfuse, seen, "raw_gen_ai_request", "inside the caller span") - await _complete(upstream, "no caller span") - exported: Final = await _exported_until(langfuse, seen, "litellm_request", "no caller span") - - (app_span,) = app_exporter.get_finished_spans() - app_attributes: Final[Mapping[str, object]] = app_span.attributes or {} - assert app_attributes["langfuse.observation.type"] == "generation", sorted(app_attributes) - assert not [key for key in app_attributes if _is_identity(key)], sorted(app_attributes) - (own_span,) = [attributes for name, attributes in exported if name == "litellm_request"] - assert json.loads(str(own_span["langfuse.observation.metadata"])) == _IDENTITY, own_span - assert {key: value for key, value in own_span.items() if key.startswith("langfuse.trace.metadata.")} == { - f"langfuse.trace.metadata.{field}": value for field, value in _IDENTITY.items() - }, sorted(own_span) diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index fee996a5d1d..9f536ab0dac 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -1,20 +1,13 @@ import base64 import json import os -from collections.abc import Mapping -from contextlib import nullcontext from datetime import datetime, timezone -from types import MappingProxyType from typing import Final from unittest.mock import MagicMock, patch import pytest -from opentelemetry import trace -from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace import TracerProvider -import litellm from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.types.llms.openai import ResponsesAPIResponse @@ -519,313 +512,6 @@ class TestLangfuseOtelIntegration: assert isinstance(config, OpenTelemetryConfig) # Endpoint assertion removed as side effect is gone - def test_request_metadata_is_emitted_under_langfuse_observation_and_trace_metadata_keys(self) -> None: - request_metadata: Final = { - "user_api_key_hash": "hash123", - "user_api_key_alias": "prod-key", - "user_api_key_user_id": "user-1", - "user_api_key_end_user_id": "end-user-1", - "user_api_key_team_id": "team-1", - "user_api_key_team_alias": "team-a", - "user_api_key_org_id": "org-1", - "spend_logs_metadata": {"env": "prod"}, - "requester_ip_address": "10.0.0.1", - "requester_metadata": {"ticket": "LIT-8283"}, - } - kwargs: Final[dict[str, object]] = { - "litellm_params": {"metadata": {}}, - "standard_logging_object": {"metadata": request_metadata}, - } - - with ( - patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, - patch("litellm.integrations.arize._utils.set_attributes"), - ): - LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) - - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - - assert json.loads(actual["langfuse.observation.metadata"]) == { - "user_api_key_alias": "prod-key", - "user_api_key_user_id": "user-1", - "user_api_key_end_user_id": "end-user-1", - "user_api_key_team_id": "team-1", - "user_api_key_team_alias": "team-a", - "user_api_key_org_id": "org-1", - } - assert {key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.")} == { - "langfuse.trace.metadata.user_api_key_alias": "prod-key", - "langfuse.trace.metadata.user_api_key_user_id": "user-1", - "langfuse.trace.metadata.user_api_key_end_user_id": "end-user-1", - "langfuse.trace.metadata.user_api_key_team_id": "team-1", - "langfuse.trace.metadata.user_api_key_team_alias": "team-a", - } - - def test_request_metadata_redaction_matches_vanilla_langfuse(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "redact_user_api_key_info", True) - request_metadata: Final = { - "user_api_key_hash": "hash123", - "user_api_key_alias": "prod-key", - "user_api_key_user_id": "user-1", - "user_api_key_end_user_id": "end-user-1", - "user_api_key_team_id": "team-1", - "user_api_key_team_alias": "team-a", - "spend_logs_metadata": {"env": "prod"}, - "requester_ip_address": "10.0.0.1", - "requester_metadata": {"ticket": "LIT-8283"}, - } - kwargs: Final[dict[str, object]] = { - "litellm_params": {"metadata": {}}, - "standard_logging_object": {"metadata": request_metadata}, - } - - with ( - patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, - patch("litellm.integrations.arize._utils.set_attributes"), - ): - LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) - - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - - assert "langfuse.observation.metadata" not in actual - assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] - - def test_request_metadata_beyond_the_identity_fields_stays_off_the_langfuse_metadata_keys(self) -> None: - request_metadata: Final = { - "user_api_key_alias": "prod-key", - "requester_ip_address": "10.0.0.1", - "user_api_key_user_email": "caller-email-secret", - "user_api_key_auth_metadata": {"note": "auth-metadata-secret"}, - "user_agent": "user-agent-secret", - "headers": {"x-tenant-secret": "header-secret"}, - "requester_custom_headers": {"x-tenant-secret": "custom-header-secret"}, - "requester_metadata": {"note": "requester-metadata-secret"}, - "spend_logs_metadata": {"note": "spend-logs-secret"}, - "prompt_management_metadata": {"prompt_variables": {"input": "prompt-variables-secret"}}, - "mcp_tool_call_metadata": {"arguments": {"location": "mcp-arguments-secret"}}, - "vector_store_request_metadata": [{"query": "vector-query-secret"}], - "routing_decision": {"matched_keyword": "routing-prompt-secret"}, - } - kwargs: Final[dict[str, object]] = { - "litellm_params": {"metadata": {}}, - "standard_logging_object": {"metadata": request_metadata}, - } - - with ( - patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, - patch("litellm.integrations.arize._utils.set_attributes"), - ): - LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) - - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - langfuse_metadata: Final = { - key: value - for key, value in actual.items() - if key == "langfuse.observation.metadata" or key.startswith("langfuse.trace.metadata.") - } - - assert langfuse_metadata == { - "langfuse.observation.metadata": json.dumps({"user_api_key_alias": "prod-key"}), - "langfuse.trace.metadata.user_api_key_alias": "prod-key", - } - - def test_request_metadata_drops_empty_fields_and_skips_empty_trace_identities(self) -> None: - request_metadata: Final = { - "user_api_key_alias": "prod-key", - "user_api_key_user_id": "user-1", - "user_api_key_end_user_id": None, - "user_api_key_team_id": "", - "user_api_key_team_alias": None, - "team_id": None, - "team_alias": None, - "spend_logs_metadata": {"env": "prod"}, - } - kwargs: Final[dict[str, object]] = { - "litellm_params": {"metadata": {}}, - "standard_logging_object": {"metadata": request_metadata}, - } - - with ( - patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, - patch("litellm.integrations.arize._utils.set_attributes"), - ): - LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) - - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - - assert json.loads(actual["langfuse.observation.metadata"]) == { - "user_api_key_alias": "prod-key", - "user_api_key_user_id": "user-1", - } - assert {key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.")} == { - "langfuse.trace.metadata.user_api_key_alias": "prod-key", - "langfuse.trace.metadata.user_api_key_user_id": "user-1", - } - - def test_request_metadata_keys_are_absent_without_identity_fields(self) -> None: - kwargs: Final[dict[str, object]] = { - "litellm_params": {"metadata": {}}, - "standard_logging_object": { - "metadata": {"spend_logs_metadata": {"env": "prod"}, "requester_ip_address": "10.0.0.1"} - }, - } - - with ( - patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, - patch("litellm.integrations.arize._utils.set_attributes"), - ): - LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) - - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - - assert "langfuse.observation.metadata" not in actual - assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")] - - def test_request_metadata_keys_are_absent_without_standard_logging_object(self) -> None: - kwargs: Final = {"litellm_params": {"metadata": {"trace_metadata": {"k": "v"}}}} - - with ( - patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, - patch("litellm.integrations.arize._utils.set_attributes"), - ): - LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, None) - - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - - assert [key for key in actual if key.startswith("langfuse.trace.metadata")] == ["langfuse.trace.metadata"] - assert "langfuse.observation.metadata" not in actual - - @pytest.mark.parametrize( - "unexpected_shape", - [ - {"prompt_management_metadata": {1: "one", "prompt_id": "prompt-1"}}, - {"mcp_tool_call_metadata": {1: "one", "name": "lookup_weather"}}, - {"vector_store_request_metadata": [{1: "one", "vector_store_id": "vs-1"}]}, - {"requester_metadata": 1}, - {"requester_metadata": True}, - {"requester_metadata": 1.5}, - {1: "one"}, - {"user_api_key_user_id": 5}, - {"user_api_key_team_id": {"id": "team-1"}}, - ], - ) - def test_request_metadata_of_unexpected_shape_keeps_the_observation_output( - self, unexpected_shape: dict[object, object] - ) -> None: - kwargs: Final[dict[str, object]] = { - "litellm_params": {"metadata": {}}, - "standard_logging_object": {"metadata": {**unexpected_shape, "user_api_key_alias": "prod-key"}}, - } - response_obj: Final = {"choices": [{"message": {"role": "assistant", "content": "hello"}}]} - - with ( - patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute, - patch("litellm.integrations.arize._utils.set_attributes"), - ): - LangfuseOtelLogger.set_langfuse_otel_attributes(MagicMock(), kwargs, response_obj) - - actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list} - - assert json.loads(actual["langfuse.observation.metadata"]) == {"user_api_key_alias": "prod-key"} - assert json.loads(actual["langfuse.observation.output"]) == {"role": "assistant", "content": "hello"} - - # What ``OpenTelemetry._record_exception_on_span`` adds to a failed call's span after its attributes. - _FAILED_CALL_ATTRIBUTES: Final = MappingProxyType( - { - "error.code": "429", - "http.response.status_code": 429, - "error.type": "RateLimitError", - "error.message": "rate limited", - "error.llm_provider": "openai", - "error.stack_trace": "Traceback", - } - ) - - @staticmethod - def _exported_span( - kwargs: dict[str, object], - max_span_attributes: int | None, - carried: dict[str, str] | None = None, - added_after: Mapping[str, str | int] = MappingProxyType({}), - active: bool = False, - ) -> ReadableSpan: - exporter: Final = InMemorySpanExporter() - provider: Final = TracerProvider(span_limits=SpanLimits(max_span_attributes=max_span_attributes)) - provider.add_span_processor(SimpleSpanProcessor(exporter)) - span: Final = provider.get_tracer("langfuse-otel-test").start_span("litellm_request", attributes=carried) - response_obj: Final = {"id": "chatcmpl-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]} - with trace.use_span(span, end_on_exit=False) if active else nullcontext(): - LangfuseOtelLogger.set_langfuse_otel_attributes(span, kwargs, response_obj) - span.set_attributes(added_after) - span.end() - (finished,) = exporter.get_finished_spans() - return finished - - @staticmethod - def _tool_call_kwargs() -> dict[str, object]: - tools: Final = [ - { - "type": "function", - "function": {"name": f"tool_{n}", "description": f"does thing {n}", "parameters": {"type": "object"}}, - } - for n in range(3) - ] - messages: Final = [{"role": "user", "content": "hello"}] - return { - "model": "gpt-4o", - "messages": messages, - "optional_params": {"tools": tools}, - "litellm_params": {"metadata": {}}, - "standard_logging_object": { - "model": "gpt-4o", - "custom_llm_provider": "openai", - "call_type": "completion", - "messages": messages, - "model_parameters": {"tools": tools}, - "metadata": {"user_api_key_alias": "prod-key", "user_api_key_team_id": "team-1"}, - }, - } - - def test_request_metadata_never_evicts_or_drops_another_attribute_at_the_span_limit( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - kwargs: Final = self._tool_call_kwargs() - failed: Final = self._FAILED_CALL_ATTRIBUTES - monkeypatch.setattr(litellm, "redact_user_api_key_info", True) - plain: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET, added_after=failed).attributes or {}) - succeeded: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET).attributes or {}) - monkeypatch.setattr(litellm, "redact_user_api_key_info", False) - full: Final = dict(self._exported_span(kwargs, SpanLimits.UNSET, added_after=failed).attributes or {}) - identity_keys: Final = set(full) - set(plain) - assert identity_keys == { - "langfuse.observation.metadata", - "langfuse.trace.metadata.user_api_key_alias", - "langfuse.trace.metadata.user_api_key_team_id", - } - - kept_identity: list[int] = [] # mutable-ok: one count per limit - for limit in range(len(plain), len(full) + 1): - failed_span = self._exported_span(kwargs, limit, added_after=failed) - succeeded_span = self._exported_span(kwargs, limit - len(failed)) - for span, expected in ((failed_span, plain), (succeeded_span, succeeded)): - attributes = dict(span.attributes or {}) - assert span.dropped_attributes == 0 - assert {key: attributes.get(key) for key in expected} == expected - assert set(attributes) - set(expected) <= identity_keys - kept_identity.append(len(set(failed_span.attributes or {}) & identity_keys)) - assert kept_identity == list(range(len(identity_keys) + 1)) - - @pytest.mark.parametrize("carried", [None, {"caller.attribute": "kept"}], ids=["bare", "with_attributes"]) - def test_request_metadata_skips_the_callers_active_span(self, carried: dict[str, str] | None) -> None: - attributes: Final = dict( - self._exported_span(self._tool_call_kwargs(), SpanLimits.UNSET, carried, active=True).attributes or {} - ) - - assert {key: attributes[key] for key in carried or {}} == (carried or {}) - assert attributes["langfuse.observation.type"] == "generation" - assert "langfuse.observation.metadata" not in attributes - assert not [key for key in attributes if key.startswith("langfuse.trace.metadata.")] - class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target @@ -1405,38 +1091,3 @@ def test_langfuse_otel_does_not_start_proxy_request_span() -> None: finally: langfuse_provider.shutdown() generic_provider.shutdown() - - -def test_langfuse_otel_success_writes_identity_onto_its_own_span_and_never_onto_the_callers() -> None: - langfuse_exporter: Final = InMemorySpanExporter() - langfuse_provider: Final = TracerProvider() - langfuse_provider.add_span_processor(SimpleSpanProcessor(langfuse_exporter)) - app_exporter: Final = InMemorySpanExporter() - app_provider: Final = TracerProvider() - app_provider.add_span_processor(SimpleSpanProcessor(app_exporter)) - with patch.dict(os.environ, LANGFUSE_ENV_ONLY, clear=True): - logger: Final = LangfuseOtelLogger(callback_name="langfuse_otel", tracer_provider=langfuse_provider) - response_obj: Final = {"id": "chatcmpl-1", "choices": [{"message": {"role": "assistant", "content": "hello"}}]} - started_at: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) - try: - with app_provider.get_tracer("app").start_as_current_span("app.operation"): - logger._handle_success( - TestLangfuseOtelIntegration._tool_call_kwargs(), response_obj, started_at, started_at - ) - logger._handle_success(TestLangfuseOtelIntegration._tool_call_kwargs(), response_obj, started_at, started_at) - - (app_span,) = app_exporter.get_finished_spans() - (own_span,) = [span for span in langfuse_exporter.get_finished_spans() if span.name == "litellm_request"] - app_attributes: Final = dict(app_span.attributes or {}) - own_attributes: Final = dict(own_span.attributes or {}) - assert app_attributes["langfuse.observation.type"] == "generation" - identity_prefixes: Final = ("langfuse.observation.metadata", "langfuse.trace.metadata.") - assert not [key for key in app_attributes if key.startswith(identity_prefixes)] - assert json.loads(str(own_attributes["langfuse.observation.metadata"])) == { - "user_api_key_alias": "prod-key", - "user_api_key_team_id": "team-1", - } - assert own_attributes["langfuse.trace.metadata.user_api_key_team_id"] == "team-1" - finally: - langfuse_provider.shutdown() - app_provider.shutdown() From c15e8c862b24573e37c3b56e5e8ffbb5919e85f9 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 20 Oct 2026 18:59:28 -0700 Subject: [PATCH 28/28] test(langfuse otel): count only accepted batches in the sink outage cell The wire also records the batches the sink answered 503, so an exporter retry made a span look exported twice. The chaos cells now also check the spans that landed carry the caller identity. --- .../test_langfuse_otel_metadata.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/integration/observability/test_langfuse_otel_metadata.py b/tests/integration/observability/test_langfuse_otel_metadata.py index 78b31d8ee4e..46fca44535f 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata.py +++ b/tests/integration/observability/test_langfuse_otel_metadata.py @@ -3,6 +3,7 @@ import binascii import json import threading import uuid +from collections import Counter from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor, wait from contextlib import contextmanager @@ -298,6 +299,12 @@ def _watched_spans(collector: Wire) -> Callable[[], tuple[dict[str, object], ... return snapshot +def _assert_key_alias(spans: Sequence[Mapping[str, object]], key_alias: str | None) -> None: + assert spans, "no span landed" + for span in spans: + assert json.loads(str(span["langfuse.observation.metadata"]))["user_api_key_alias"] == key_alias, span + + def _v2_llm_span(collector: Wire, response_id: str) -> dict[str, object]: def matches(span: Mapping[str, object]) -> bool: return _response_id(span.get("gen_ai.response.id")) == _response_id(response_id) @@ -677,11 +684,11 @@ def test_langfuse_otel_repeated_identical_requests_emit_one_span_each(gateway: G def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "lf-chaos-outage-" + uuid.uuid4().hex outage: Final = threading.Event() - rejected: Final = [] # mutable-ok: the wire server thread records each 503 it serves + rejected: Final[list[bytes]] = [] # mutable-ok: the wire server thread records each batch it answers 503 def sink_reply(request: Request) -> Reply: if outage.is_set(): - rejected.append(1) + rejected.append(request.body) return Reply(status=503, body=b'{"error": "sink outage"}') return Reply() @@ -691,7 +698,13 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa ): key, identity = _scenario_identity(scenario, rig.provider) anthropic_model: Final = scenario.model(model="anthropic/claude-sonnet-4-6", api_base=rig.provider.url) - watch: Final = _watched_spans(rig.collector) + received: Final[list[bytes]] = [] # mutable-ok: drain() consumes the wire queue, so batches accumulate + + def watch() -> tuple[dict[str, object], ...]: + # The wire also records the batches the sink rejected; only the accepted ones landed. + received.extend(batch.body for batch in rig.collector.drain()) + accepted: Final = Counter(received) - Counter(rejected) + return tuple(attributes for body in accepted.elements() for attributes in _span_attributes(body)) def call(index: int) -> str: path: Final = ("/v1/chat/completions", "/v1/messages", "/v1/responses")[index % 3] @@ -756,6 +769,7 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa ) assert landed == sorted(response_ids), sorted(set(response_ids) - set(landed)) assert len(rejected) >= 1, "sink outage never served a 503" + _assert_key_alias([span for span in settled if _span_response_id(span) in response_ids], identity["key_alias"]) def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: Gateway, tmp_path: Path) -> None: @@ -805,6 +819,7 @@ def test_langfuse_otel_worker_kill_mid_burst_keeps_serving_and_exports(gateway: ) for rid in succeeded: assert sum(_span_response_id(s) == rid for s in settled) <= 1, rid + _assert_key_alias([span for span in settled if _span_response_id(span) in post_kill_ids], identity["key_alias"]) def test_langfuse_otel_slow_sink_does_not_deadlock_exports(gateway: Gateway, tmp_path: Path) -> None: @@ -844,6 +859,7 @@ def test_langfuse_otel_slow_sink_does_not_deadlock_exports(gateway: Gateway, tmp assert sorted(_span_response_id(span) for span in settled if _span_response_id(span) in response_ids) == sorted( response_ids ) + _assert_key_alias([span for span in settled if _span_response_id(span) in response_ids], identity["key_alias"]) def test_langfuse_otel_v2_mapper_emits_request_metadata_and_identity(gateway: Gateway, tmp_path: Path) -> None: