fix: avoid counting video urls as text

This commit is contained in:
pragnyanramtha 2026-05-17 15:42:18 +00:00
parent ad9e11ab21
commit 625880c9f0
2 changed files with 16 additions and 6 deletions

View file

@ -724,12 +724,7 @@ def _count_content_list(
image_url, use_default_image_token_count
)
elif c["type"] == "video_url":
video_url = c.get("video_url", "")
if isinstance(video_url, dict):
video_url = video_url.get("url")
if video_url is None:
video_url = ""
num_tokens += DEFAULT_IMAGE_TOKEN_COUNT + count_function(str(video_url))
num_tokens += DEFAULT_IMAGE_TOKEN_COUNT
elif c["type"] in ("tool_use", "tool_result"):
num_tokens += _count_anthropic_content(
c,

View file

@ -948,6 +948,21 @@ def test_token_counter_with_video_url():
tokens_str > DEFAULT_IMAGE_TOKEN_COUNT
), f"Expected default video token budget, got {tokens_str}"
messages_base64_str = [
{
"role": "user",
"content": [
{
"type": "video_url",
"video_url": "data:video/mp4;base64," + ("A" * 4000),
}
],
}
]
assert (
token_counter(model="gpt-4o", messages=messages_base64_str) == tokens_str
)
messages_empty_url = [
{
"role": "user",