diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 52d8d8c06f3..9b3ad174eda 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -29,9 +29,10 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( ) from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.llms.custom_httpx.http_handler import _get_httpx_client +from litellm.responses.utils import ResponseAPILoggingUtils from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.langfuse import * -from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse +from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ( EmbeddingResponse, ImageResponse, @@ -738,7 +739,18 @@ class LangFuseLogger: if response_obj is not None: if hasattr(response_obj, "id") and response_obj.get("id", None) is not None: generation_id = _logging_id(start_time, response_obj) - _usage_obj: Final[_UsageObject | None] = getattr(response_obj, "usage", None) + _raw_usage_obj: Final = getattr(response_obj, "usage", None) + _usage_obj: Final[_UsageObject | None] = ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # same shared transform litellm_logging uses + _raw_usage_obj + ) + if isinstance(_raw_usage_obj, ResponseAPIUsage) + or ( + isinstance(_raw_usage_obj, dict) + and ResponseAPILoggingUtils._is_response_api_usage(_raw_usage_obj) # pyright: ignore[reportPrivateUsage] # no public wrapper + ) + else _raw_usage_obj + ) if _usage_obj: # Safely get usage values, defaulting None to 0 for Langfuse compatibility. diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index 1f2f1d64711..2cdc15c4938 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -25,3 +25,4 @@ - {id: logging.niche_integrations.success.logs_spend, module: logging, tier: P2, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE cohort: athina/galileo/deepeval/langtrace/weave/lunary/humanloop/traceloop/helicone/argilla/newrelic/sqs/supabase/dynamodb/agentops/lago/etc"} - {id: logging.niche_integrations.failure.logs_spend, module: logging, tier: P2, event: failure, assertions: [logs_spend], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE niche failure path"} - {id: logging.langfuse.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/langfuse/langfuse_otel.py", rationale: "Team-scoped Langfuse delivery via /team/callback; LangChain-ecosystem evals spend"} +- {id: logging.langfuse.success.logs_usage, module: logging, tier: P1, event: success, assertions: [logs_usage], exercised_on: [responses], source: "integrations/langfuse/langfuse.py", rationale: "Classic Langfuse SDK generations must carry token usage on the /v1/responses path; ResponseAPIUsage read as chat Usage logs all zeros (LIT-8238)"} diff --git a/tests/e2e/logging/test_responses_langfuse_usage_e2e.py b/tests/e2e/logging/test_responses_langfuse_usage_e2e.py new file mode 100644 index 00000000000..cf31de24250 --- /dev/null +++ b/tests/e2e/logging/test_responses_langfuse_usage_e2e.py @@ -0,0 +1,111 @@ +"""Live e2e: Langfuse token usage on the /v1/responses path. + +Covers logging.langfuse.success.logs_usage: a key-scoped ``langfuse`` +(classic SDK) callback must log non-zero input/output tokens to the Langfuse +generation for a /v1/responses call. A ResponsesAPIResponse carries +ResponseAPIUsage (input_tokens/output_tokens), which the logger must +normalize before building usage_details; without the normalization Langfuse +shows 0 input / 0 output while cost is still right. +""" + +from __future__ import annotations + +import pytest +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from lifecycle import ResourceManager +from logging_client import ( + LangfuseCreds, + LoggingClient, + first_ok, + load_langfuse_creds, +) +from models import ( + KeyLoggingCallback, + KeyLoggingCallbackVars, + KeyMetadata, + ResponsesApiResponse, +) + +pytestmark = pytest.mark.e2e + + +class _LangfuseUsageDetails(BaseModel): + model_config = ConfigDict(extra="ignore") + + input: int | None = None + output: int | None = None + total: int | None = None + cache_read_input_tokens: int | None = None + + +_USAGE_DETAILS_ADAPTER: TypeAdapter[_LangfuseUsageDetails] = TypeAdapter(_LangfuseUsageDetails) + + +@pytest.fixture(scope="session") +def langfuse_creds() -> LangfuseCreds: + return load_langfuse_creds() + + +class TestResponsesLangfuseUsage: + @pytest.mark.covers("logging.langfuse.success.logs_usage", exercised_on=["responses"]) + def test_responses_call_logs_nonzero_token_usage( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + key_alias = f"lf-resp-key-{unique_marker()}" + key = client.key_with_alias( + key_alias, + models=[CHEAP_OPENAI_MODEL], + metadata=KeyMetadata( + logging=[ + KeyLoggingCallback( + callback_name="langfuse", + callback_type="success", + callback_vars=KeyLoggingCallbackVars( + langfuse_public_key=langfuse_creds.public_key, + langfuse_secret_key=langfuse_creds.secret_key, + langfuse_host=langfuse_creds.host, + ), + ) + ] + ), + ) + resources.defer(lambda: client.delete_key(key)) + + marker = unique_marker() + outcome = first_ok( + client, + lambda: client.responses_raw( + key, CHEAP_OPENAI_MODEL, f"reply with one word {marker}", max_output_tokens=64 + ), + ) + response = ResponsesApiResponse.model_validate_json(outcome.body) + assert response.usage is not None and response.usage.input_tokens > 0, ( + f"the /v1/responses body must report input_tokens, got {outcome.body[:300]}" + ) + + observation = client.poll_langfuse_observation( + langfuse_creds, + key_alias=key_alias, + prompt_marker=marker, + ) + assert observation is not None, ( + f"the key's /v1/responses call (marker {marker}) never reached Langfuse within the deadline" + ) + assert observation.usage_details is not None, ( + f"the Langfuse generation must carry usageDetails, got {observation.model_dump_json()}" + ) + details = _USAGE_DETAILS_ADAPTER.validate_python(observation.usage_details) + cache_read = details.cache_read_input_tokens or 0 + assert details.input is not None and details.input > 0, ( + f"usageDetails.input must be non-zero, got {details.model_dump()}" + ) + assert details.input + cache_read == response.usage.input_tokens, ( + f"usageDetails.input {details.input} + cache_read {cache_read} must equal the proxy's " + f"input_tokens {response.usage.input_tokens}" + ) + assert details.output == response.usage.output_tokens, ( + f"usageDetails.output {details.output} must equal the proxy's " + f"output_tokens {response.usage.output_tokens}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 4b202e3c663..e534133b6c9 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -429,6 +429,26 @@ class ChatResponse(BaseModel): service_tier: str | None = None +class ResponsesInputTokensDetails(BaseModel): + cached_tokens: int | None = None + + +class ResponsesUsage(BaseModel): + """`/v1/responses` usage shape: input/output tokens, not prompt/completion.""" + + input_tokens: int + output_tokens: int + total_tokens: int | None = None + input_tokens_details: ResponsesInputTokensDetails | None = None + + +class ResponsesApiResponse(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: str | None = None + usage: ResponsesUsage | None = None + + # ---------- anthropic /v1/messages + count_tokens ---------- diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index 37860ae8445..5c0dbf288dc 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -11,6 +11,7 @@ import pytest import litellm from litellm.integrations.langfuse import langfuse as langfuse_module from litellm.integrations.langfuse.langfuse import LangFuseLogger +from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage, ResponsesAPIResponse # Import LangfuseUsageDetails directly from the module where it's defined @@ -358,6 +359,92 @@ class TestLangfuseUsageDetails(unittest.TestCase): mock_add_prompt_params.assert_called_once() + def test_log_langfuse_v2_responses_api_usage(self): + """ + Regression test: a /v1/responses response carries ResponseAPIUsage + (input_tokens/output_tokens), which must be normalized to a chat Usage + before Langfuse usage_details are read, or generations log 0 tokens. + """ + self.mock_langfuse_client.reset_mock(side_effect=True) + self.mock_langfuse_trace.reset_mock(side_effect=True) + self.mock_langfuse_generation.reset_mock(side_effect=True) + + self.mock_langfuse_generation.trace_id = "test-trace-id" + mock_span = MagicMock() + mock_span.end = MagicMock() + self.mock_langfuse_trace.span.return_value = mock_span + self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation + self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace + self.logger.Langfuse = self.mock_langfuse_client + + with ( + patch( + "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", + side_effect=lambda generation_params, **kwargs: generation_params, + create=True, + ), + patch.object(self.logger, "_supports_prompt", return_value=True), + ): + response_obj = ResponsesAPIResponse( + id="resp_123", + created_at=0, + output=[], + usage=ResponseAPIUsage( + input_tokens=16, + output_tokens=21, + total_tokens=37, + input_tokens_details=InputTokensDetails(cached_tokens=4), + ), + ) + + kwargs = { + "model": "gpt-5.5", + "messages": [{"role": "user", "content": "Test"}], + "litellm_params": {"metadata": {}}, + "optional_params": {}, + "litellm_call_id": "test-call-id-responses-api-usage", + "standard_logging_object": self._build_standard_logging_payload(), + "response_cost": 0.0, + } + + fixed_time = datetime.datetime(2024, 1, 1, 12, 0, 0) + + try: + self.logger._log_langfuse_v2( + user_id="test-user", + metadata={}, + litellm_params=kwargs["litellm_params"], + output={"role": "assistant", "content": "Response"}, + start_time=fixed_time, + end_time=fixed_time + datetime.timedelta(seconds=1), + kwargs=kwargs, + optional_params=kwargs["optional_params"], + input={"messages": kwargs["messages"]}, + response_obj=response_obj, + level="DEFAULT", + litellm_call_id=kwargs["litellm_call_id"], + ) + except Exception as e: + self.fail(f"_log_langfuse_v2 raised an exception: {e}") + + self.mock_langfuse_trace.generation.assert_called_once() + call_args, call_kwargs = self.mock_langfuse_trace.generation.call_args + + usage_arg = call_kwargs.get("usage") + usage_details_arg = call_kwargs.get("usage_details") + + self.assertIsNotNone(usage_arg) + self.assertIsNotNone(usage_details_arg) + + self.assertEqual(usage_arg["prompt_tokens"], 16) + self.assertEqual(usage_arg["completion_tokens"], 21) + + # input is reduced by cache_read_input_tokens per Langfuse docs + self.assertEqual(usage_details_arg["input"], 12) + self.assertEqual(usage_details_arg["output"], 21) + self.assertEqual(usage_details_arg["total"], 37) + self.assertEqual(usage_details_arg["cache_read_input_tokens"], 4) + def _build_standard_logging_payload(self, trace_id: Optional[str] = None): payload = { "id": "payload-id",