Merge 8d186bcad5 into 587f227b9d

litellm/litellm_core_utils/token_counter.py

@@ -738,11 +738,19 @@ def _count_content_list(
                 tool_name = str(c.get("tool_name") or "")
                 if tool_name:
                     num_tokens += count_function(tool_name)
+            elif c["type"] in ("video_url", "input_audio", "file"):
+                # Opaque payloads: a video or audio data URI, or an uploaded file
+                # reference. The provider decides their cost (frame sampling rate,
+                # audio chunking), so they contribute 0 rather than raise - on the
+                # ollama route the counter runs after generation, where raising
+                # discards a response the model already produced.
+                pass
             else:
                 content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__
                 raise ValueError(
                     f"Invalid content item type: {content_type}. "
-                    f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)."
+                    f"Expected str or dict with 'type' field (text, image_url, video_url, input_audio, "
+                    f"file, tool_use, tool_result, thinking, tool_reference)."
                 )
         return num_tokens
     except Exception as e:

tests/test_litellm/litellm_core_utils/test_token_counter.py

@@ -1160,3 +1160,60 @@ def test_count_content_list_rejects_unknown_type():
     message = str(exc_info.value)
     assert "Invalid content item type: totally_unknown_block" in message
     assert "tool_reference" in message
+
+
+@pytest.mark.parametrize(
+    "content_block",
+    [
+        {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AAAA"}},
+        {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "mp3"}},
+        {
+            "type": "file",
+            "file": {
+                "file_data": "data:application/pdf;base64,AAAA",
+                "filename": "report.pdf",
+            },
+        },
+    ],
+    ids=["video_url", "input_audio", "file"],
+)
+def test_token_counter_with_non_text_modality_blocks(content_block: dict):
+    """
+    Regression test: video / audio / file content blocks must NOT raise.
+
+    Before the fix, token_counter raised e.g.
+    `Invalid content item type: video_url`. Two paths break as a result: on the
+    proxy this nulls response_cost and drops the SpendLogs row, and on the
+    ollama route the counter runs after generation, so an already-produced 200
+    response is turned into a 500.
+
+    Their payload is opaque here, so they contribute 0 tokens and the count
+    matches the same message without the block.
+    """
+    prompt = {"type": "text", "text": "Describe the attachment."}
+    messages = [{"role": "user", "content": [prompt, content_block]}]
+    text_only = [{"role": "user", "content": [prompt]}]
+
+    tokens = token_counter_new(model="gpt-4o", messages=messages)
+    assert tokens > 0, f"Expected positive token count, got {tokens}"
+    assert tokens == token_counter_new(model="gpt-4o", messages=text_only)
+
+
+def test_count_content_list_error_message_lists_modality_types():
+    """
+    The catch-all error must enumerate the handled block types so a future type
+    is not silently dropped, and the non-text modality types must appear there.
+    """
+    from litellm.litellm_core_utils.token_counter import _count_content_list
+
+    with pytest.raises(ValueError, match="Invalid content item type: totally_unknown_block") as exc_info:
+        _count_content_list(
+            count_function=len,
+            content_list=[{"type": "totally_unknown_block"}],
+            use_default_image_token_count=False,
+            default_token_count=None,
+        )
+
+    message = str(exc_info.value)
+    for content_type in ("video_url", "input_audio", "file"):
+        assert content_type in message

tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py

@@ -904,3 +904,70 @@ class TestOllamaToolCallTransformation:
         assert tool_msg["content"] == "Sunny, 72°F"
         assert "tool_call_id" in tool_msg, "tool_call_id must be forwarded to Ollama"
         assert tool_msg["tool_call_id"] == "call_abc123"
+
+
+class TestOllamaChatVideoBlockUsage:
+    """Usage counting for requests that carry a non-text content block."""
+
+    @staticmethod
+    def _transform(
+        config: OllamaChatConfig,
+        ollama_response: dict,
+        messages: list[AllMessageValues],
+    ) -> ModelResponse:
+        import json
+        from unittest.mock import MagicMock
+
+        from litellm.types.utils import Choices, Message, ModelResponse
+
+        mock_response = MagicMock()
+        mock_response.json.return_value = ollama_response
+        mock_response.text = json.dumps(ollama_response)
+
+        model_response = ModelResponse()
+        model_response.choices = [Choices(message=Message(content=""), index=0)]
+
+        return config.transform_response(
+            model="qwen3:14b",
+            raw_response=mock_response,
+            model_response=model_response,
+            logging_obj=MagicMock(),
+            request_data={},
+            messages=messages,
+            optional_params={},
+            litellm_params={},
+            encoding=None,
+            api_key=None,
+            json_mode=False,
+        )
+
+    def test_missing_counts_with_a_video_block_do_not_raise(self):
+        """
+        End-to-end of the two fixes on this route: Ollama omits the counts, so the
+        estimator runs over a message that carries a `video_url` block. It must
+        produce usage instead of raising, because the response has already been
+        generated by the time the counter runs.
+        """
+        ollama_response = {
+            "model": "qwen3:14b",
+            "created_at": "2025-01-11T00:00:00.000000Z",
+            "message": {"role": "assistant", "content": "A short clip."},
+            "done": True,
+        }
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Describe this."},
+                    {
+                        "type": "video_url",
+                        "video_url": {"url": "data:video/mp4;base64,AAAA"},
+                    },
+                ],
+            }
+        ]
+
+        result = self._transform(OllamaChatConfig(), ollama_response, messages)
+
+        assert result.usage.prompt_tokens > 0
+        assert result.usage.completion_tokens > 0