From 2c239480726efe938bdd2a4f1bd47557156c083c Mon Sep 17 00:00:00 2001 From: Rodrigo Azevedo Date: Tue, 27 Oct 2026 10:39:57 -0300 Subject: [PATCH 1/2] fix(langfuse): trace successful decisions calls with their token counts Co-Authored-By: Claude Opus 5.5 --- litellm/integrations/langfuse/langfuse.py | 17 ++++++-- .../integrations/langfuse/test_langfuse.py | 43 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index b8c6e91d0d7..825f1b89e90 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -15,6 +15,7 @@ from packaging.version import Version import litellm from litellm._logging import verbose_logger from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS +from litellm.cost_calculator import get_usage_object from litellm.integrations.langfuse.langfuse_mock_client import ( create_mock_langfuse_client, should_use_langfuse_mock, @@ -32,6 +33,7 @@ from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.llms.custom_httpx.http_handler import get_httpx_client from litellm.responses.utils import ResponseAPILoggingUtils from litellm.secret_managers.main import str_to_bool +from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse from litellm.types.integrations.langfuse import * from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ( @@ -584,7 +586,9 @@ class LangFuseLogger: | TranscriptionResponse | RerankResponse | HttpxBinaryResponseContent - | ResponsesAPIResponse, + | ResponsesAPIResponse + | DecisionsResponse + | OpenAIDecisionResponse, prompt: dict, level: str, status_message: str | None, @@ -634,6 +638,9 @@ class LangFuseLogger: elif response_obj is not None and isinstance(response_obj, litellm.ResponsesAPIResponse): input = prompt output = self._get_responses_api_content_for_langfuse(response_obj) + elif response_obj is not None and isinstance(response_obj, (DecisionsResponse, OpenAIDecisionResponse)): + input = prompt + output = response_obj.model_dump(mode="json", exclude={"usage"}) elif ( kwargs.get("call_type") is not None and kwargs.get("call_type") == "_arealtime" @@ -859,10 +866,14 @@ class LangFuseLogger: usage = None usage_details = None if response_obj is not None: - if hasattr(response_obj, "id") and response_obj.get("id", None) is not None: + if isinstance(response_obj, _ResponseWithId) and getattr(response_obj, "id", None) is not None: generation_id = _logging_id(start_time, response_obj) _raw_usage_obj: Final[_UsageObject | None] = getattr(response_obj, "usage", None) - _usage_obj: Final = _chat_usage(_raw_usage_obj) + _usage_obj: Final = ( + get_usage_object(response_obj) + if isinstance(response_obj, (DecisionsResponse, OpenAIDecisionResponse)) + else _chat_usage(_raw_usage_obj) + ) if _usage_obj: # Safely get usage values, defaulting None to 0 for Langfuse compatibility. diff --git a/tests/unit/integrations/langfuse/test_langfuse.py b/tests/unit/integrations/langfuse/test_langfuse.py index 7a374c1affe..ecaecebf276 100644 --- a/tests/unit/integrations/langfuse/test_langfuse.py +++ b/tests/unit/integrations/langfuse/test_langfuse.py @@ -433,6 +433,49 @@ async def test_langfuse_logging_vllm_embedding( assert route.call_count == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize("response_id", [None, "decision-1"]) +async def test_langfuse_logging_systemone( + langfuse_export: tuple[respx.Route, str], respx_mock: respx.MockRouter, response_id: str | None +) -> None: + route, trace_id = langfuse_export + answers: Final = {"urgent": {"type": "noul", "noul": 0.87}} + upstream: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond( + json={ + **({"id": response_id} if response_id is not None else {}), + "model": "typesafe/jev-1.13", + "answers": answers, + "usage": {"input_tokens": 283, "output_tokens": 20}, + } + ) + await litellm.asystemone( + model="openrouter/typesafe/jev-1.13", + state="Customer writes: my payout has failed three days in a row.", + questions={"urgent": {"type": "noul", "instructions": "Is this urgent?"}}, + api_key="caller-key", + metadata={"trace_id": trace_id}, + ) + await _flush_langfuse() + assert upstream.call_count == 1 + generations: Final = tuple( + TypeAdapter(dict[str, object]).validate_python(span["attributes"]) + for span in _exported_spans(route) + if span["trace_id"] == resolve_trace_id(trace_id) + and TypeAdapter(dict[str, object]).validate_python(span["attributes"]).get("langfuse.observation.type") + == "generation" + ) + assert len(generations) == 1 + assert generations[0]["langfuse.observation.usage_details"] == { + "input": 283, + "output": 20, + "total": 303, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + output: Final = TypeAdapter(dict[str, object]).validate_python(generations[0]["langfuse.observation.output"]) + assert output["answers"] == answers + + @pytest.mark.asyncio async def test_langfuse_logging_with_router(langfuse_export: tuple[respx.Route, str]) -> None: route, trace_id = langfuse_export From c0753dcdb465969a12837557baa2c95d10903a25 Mon Sep 17 00:00:00 2001 From: Rodrigo Azevedo Date: Tue, 27 Oct 2026 13:18:43 -0300 Subject: [PATCH 2/2] fix(langfuse): pass lint gates and pin the decisions generation to a fixture Return the decisions input/output directly instead of rebinding them, log only the answers as output so provider extras like the response id stay out, and check the exported generation (cost, tokens, input, output) against a golden fixture like the other Langfuse tests. Co-Authored-By: Claude Opus 5.5 --- litellm/integrations/langfuse/langfuse.py | 8 ++-- .../systemone.json | 38 +++++++++++++++++++ .../integrations/langfuse/test_langfuse.py | 20 +--------- 3 files changed, 45 insertions(+), 21 deletions(-) create mode 100644 tests/unit/integrations/langfuse/langfuse_expected_request_body/systemone.json diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 825f1b89e90..ee67228841a 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -638,9 +638,11 @@ class LangFuseLogger: elif response_obj is not None and isinstance(response_obj, litellm.ResponsesAPIResponse): input = prompt output = self._get_responses_api_content_for_langfuse(response_obj) - elif response_obj is not None and isinstance(response_obj, (DecisionsResponse, OpenAIDecisionResponse)): - input = prompt - output = response_obj.model_dump(mode="json", exclude={"usage"}) + elif isinstance(response_obj, (DecisionsResponse, OpenAIDecisionResponse)): + return ( # pyright: ignore[reportUnknownVariableType] # prompt is the caller's untyped dict, as in every branch + prompt, + response_obj.model_dump(mode="json", include={"answers"}), + ) elif ( kwargs.get("call_type") is not None and kwargs.get("call_type") == "_arealtime" diff --git a/tests/unit/integrations/langfuse/langfuse_expected_request_body/systemone.json b/tests/unit/integrations/langfuse/langfuse_expected_request_body/systemone.json new file mode 100644 index 00000000000..50e1ea18848 --- /dev/null +++ b/tests/unit/integrations/langfuse/langfuse_expected_request_body/systemone.json @@ -0,0 +1,38 @@ +{ + "name": "litellm-asystemone", + "parent_span_id": null, + "attributes": { + "langfuse.observation.cost_details": { + "total": 1.1886e-05 + }, + "langfuse.observation.input": { + "messages": [ + { + "content": "{\"state\": \"Customer writes: my payout has failed three days in a row.\", \"questions\": {\"urgent\": {\"type\": \"noul\", \"instructions\": \"Is this urgent?\"}}}", + "role": "user" + } + ] + }, + "langfuse.observation.level": "DEFAULT", + "langfuse.observation.metadata.api_base": "https://openrouter.ai/api/alpha/decisions", + "langfuse.observation.model.name": "typesafe/jev-1.13", + "langfuse.observation.model.parameters": {}, + "langfuse.observation.output": { + "answers": { + "urgent": { + "noul": 0.87, + "type": "noul" + } + } + }, + "langfuse.observation.type": "generation", + "langfuse.observation.usage_details": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input": 283, + "output": 20, + "total": 303 + }, + "langfuse.trace.name": "litellm-asystemone" + } +} diff --git a/tests/unit/integrations/langfuse/test_langfuse.py b/tests/unit/integrations/langfuse/test_langfuse.py index ecaecebf276..747ebd1cc1f 100644 --- a/tests/unit/integrations/langfuse/test_langfuse.py +++ b/tests/unit/integrations/langfuse/test_langfuse.py @@ -445,7 +445,7 @@ async def test_langfuse_logging_systemone( **({"id": response_id} if response_id is not None else {}), "model": "typesafe/jev-1.13", "answers": answers, - "usage": {"input_tokens": 283, "output_tokens": 20}, + "usage": {"input_tokens": 283, "output_tokens": 20, "cost": 1.1886e-05}, } ) await litellm.asystemone( @@ -457,23 +457,7 @@ async def test_langfuse_logging_systemone( ) await _flush_langfuse() assert upstream.call_count == 1 - generations: Final = tuple( - TypeAdapter(dict[str, object]).validate_python(span["attributes"]) - for span in _exported_spans(route) - if span["trace_id"] == resolve_trace_id(trace_id) - and TypeAdapter(dict[str, object]).validate_python(span["attributes"]).get("langfuse.observation.type") - == "generation" - ) - assert len(generations) == 1 - assert generations[0]["langfuse.observation.usage_details"] == { - "input": 283, - "output": 20, - "total": 303, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - } - output: Final = TypeAdapter(dict[str, object]).validate_python(generations[0]["langfuse.observation.output"]) - assert output["answers"] == answers + _expected_generation(route, "systemone.json", trace_id) @pytest.mark.asyncio