diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 858b078d626..c44241c1a69 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -286,9 +286,12 @@ def calculate_img_tokens( if use_default_image_token_count: verbose_logger.debug("Using default image token count: %s", DEFAULT_IMAGE_TOKEN_COUNT) return DEFAULT_IMAGE_TOKEN_COUNT - if mode == "low" or mode == "auto": + if mode == "low": return base_tokens - elif mode == "high": + elif mode == "high" or mode == "auto": + # "auto" mirrors "high": actual auto-detail sizing is tile-based, so + # treating it as a flat low-detail cost silently undercounts tokens + # for the (default) case where callers never set an explicit detail. # Run the async function using the helper width, height = get_image_dimensions( data=data, diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index a2590dbca2d..b27ae2bb9c7 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -592,6 +592,44 @@ def test_img_url_token_counter(img_url, monkeypatch): assert height is not None +def test_calculate_img_tokens_auto_matches_high_not_low(monkeypatch): + """ + Regression test for the "auto" image detail mode silently being costed + as "low" detail instead of using the real tile-based sizing. + + "auto" is the *default* detail level whenever a caller doesn't set one + explicitly, and OpenAI's actual auto-detail behavior is tile-based + (same sizing math as "high"), not a flat low-res estimate. Before this + fix, `calculate_img_tokens(mode="auto")` fell into the same branch as + `mode="low"` and returned a flat 85-token estimate regardless of image + size, which meant a large image sent with no explicit detail could be + undercounted by hundreds of tokens relative to what the API actually + bills for it. + + For a 4096x4096 image this under-count is stark: "low" is a flat 85 + tokens, "high" (tile-based) is 765 tokens, and "auto" must match + "high" here, not "low". + """ + from litellm.litellm_core_utils.token_counter import calculate_img_tokens + + monkeypatch.setattr( + "litellm.litellm_core_utils.token_counter.get_image_dimensions", + lambda **kwargs: (4096, 4096), + ) + + low_tokens = calculate_img_tokens(data=b"fake", mode="low") + high_tokens = calculate_img_tokens(data=b"fake", mode="high") + auto_tokens = calculate_img_tokens(data=b"fake", mode="auto") + + assert low_tokens == 85 + assert high_tokens == 765 + assert auto_tokens == high_tokens, ( + "'auto' detail must use the same tile-based sizing as 'high', " + f"got {auto_tokens} tokens vs high={high_tokens} tokens" + ) + assert auto_tokens != low_tokens + + def test_token_encode_disallowed_special(): encode(model="gpt-3.5-turbo", text="Hello, world! <|endoftext|>") token_counter(model="gpt-3.5-turbo", text="Hello, world! <|endoftext|>")