diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index c23b3291365..d3ad7234d93 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -427,7 +427,7 @@ class LLMCallSpanData: # plain ``.get`` — no repeated ``isinstance`` guards. raw_response: Final = payload.get("response") response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}) - choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) + choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response) # ``finish_reasons`` is metadata, not content, so derive it from # ``choices_out`` before gating. The raw message/choice bodies are only # retained when content capture is enabled (see ``capture_span_content``); @@ -752,6 +752,22 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: return (choice,) +def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + markdowns: Final = tuple( + text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None + ) + if not markdowns: + return () + message: Final[_AssistantMessage] = { + "role": "assistant", + "content": "\n\n".join(markdowns), + "refusal": None, + "tool_calls": None, + } + choice: Final[_Choice] = {"message": message, "finish_reason": None} + return (choice,) + + def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None: texts: Final = tuple( text for part in parts if part.get("type") == part_type if (text := as_str(part.get(field))) is not None diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 17de3cf1e8a..01f2a13d252 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -872,6 +872,45 @@ def test_chat_choices_win_over_a_responses_output_list(): assert data.finish_reasons == ("stop",) +def _ocr_payload(pages: list[object]): + return _sample_payload( + call_type="aocr", + custom_llm_provider="mistral", + model="mistral-ocr-latest", + messages=None, + response={"object": "ocr", "model": "mistral-ocr-latest", "pages": pages, "usage_info": {"pages_processed": 2}}, + ) + + +def test_ocr_pages_become_one_assistant_choice_joined_in_page_order(): + data = LLMCallSpanData.from_standard_logging_payload( + _ocr_payload([{"index": 0, "markdown": "# Invoice"}, {"index": 1, "markdown": "Total: 42"}]), + capture_content=True, + ) + + assert data.choices_out == ( + { + "message": {"role": "assistant", "content": "# Invoice\n\nTotal: 42", "refusal": None, "tool_calls": None}, + "finish_reason": None, + }, + ) + assert data.finish_reasons == () + + +def test_ocr_output_follows_the_content_capture_gate(): + data = LLMCallSpanData.from_standard_logging_payload(_ocr_payload([{"index": 0, "markdown": "# Invoice"}])) + + assert data.choices_out == () + + +def test_ocr_pages_without_markdown_stay_empty(): + data = LLMCallSpanData.from_standard_logging_payload( + _ocr_payload([{"index": 0, "images": []}, "not-a-page"]), capture_content=True + ) + + assert data.choices_out == () + + def test_request_identity_prefers_canonical_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py index 4e375de0494..9fa198c4ec5 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py @@ -227,6 +227,28 @@ def test_langfuse_mapper_renders_a_responses_api_call_from_the_standard_logging_ assert attrs["langfuse.observation.type"] == "generation" +def test_langfuse_mapper_renders_an_ocr_call_with_the_page_markdown_as_output(): + payload = { + "call_type": "aocr", + "custom_llm_provider": "mistral", + "model": "mistral-ocr-latest", + "messages": None, + "response": { + "object": "ocr", + "model": "mistral-ocr-latest", + "pages": [{"index": 0, "markdown": "# Invoice"}, {"index": 1, "markdown": "Total: 42"}], + "usage_info": {"pages_processed": 2}, + }, + } + data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True) + attrs = LangfuseMapper().map(data) + + assert json.loads(attrs["langfuse.observation.output"]) == [ + {"role": "assistant", "content": "# Invoice\n\nTotal: 42", "refusal": None, "tool_calls": None} + ] + assert attrs["langfuse.observation.type"] == "generation" + + # --------------------------------------------------------------------------- # # Weave # --------------------------------------------------------------------------- #