From feffb6226673403caa9e25d5d8367b453f396ff0 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 23:24:36 -0700 Subject: [PATCH 01/22] test: refresh the suites that drifted from langfuse and OpenAI's retired Assistants API Two unrelated causes, both leaving staging red with tests that no longer describe anything true. #38264 gave LangFuseLogger a langfuse_environment argument and started carrying it in the credentials dict. The handler test's fake logger did not accept the new keyword, so constructing it raised TypeError, and four cases in test_langfuse_unit_tests rebuilt the cache key by hand from three fields and missed on the four-field key production now writes. Caching itself was never broken: the handler sets and gets with the same dict. The fake now takes the argument and asserts it is forwarded, and the cache assertion issues a second identical request and expects the same logger back, which is the behaviour that matters and cannot rot the next time a credential field is added. OpenAI has retired the Assistants API. /v1/assistants and /v1/threads both answer 404 with a valid key, where every live route answers 401, so nothing calling them can pass again. test_custom_logger_passthrough covered generic passthrough logging and only used assistants because it is a route with no provider-specific handler; it moves to /v1/moderations, which is still unclaimed by _is_supported_openai_endpoint, so the same generic branch is exercised. The two tests there asserted the same thing against different dead routes, so they collapse into one. The Ruby suite existed solely to drive assistants, threads, messages and runs, so it goes along with the RVM and bundler steps that were installed only to run it, and the two dead OpenAI assistants cases leave test_openai_assistants_passthrough. The Azure assistants case in that file stays. Azure runs its own lifecycle and I could not reach the CI deployment to check whether that API is still there. --- .circleci/config.yml | 39 -------- .../test_langfuse_dynamic_credentials.py | 4 + .../test_langfuse_unit_tests.py | 14 +-- .../ruby_passthrough_tests/Gemfile | 4 - .../ruby_passthrough_tests/Gemfile.lock | 42 -------- .../openai_assistants_passthrough_spec.rb | 96 ------------------ .../test_openai_assistants_passthrough.py | 52 ---------- .../test_custom_logger_passthrough.py | 97 +++---------------- 8 files changed, 21 insertions(+), 327 deletions(-) delete mode 100644 tests/pass_through_tests/ruby_passthrough_tests/Gemfile delete mode 100644 tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock delete mode 100644 tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index 4615a6a5a7e..55fa9410845 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2421,45 +2421,6 @@ jobs: - wait_for_service: url: http://localhost:4000 timeout: "300" - # Add Ruby installation and testing before the existing Node.js and Python tests - - run: - name: Install Ruby and Bundler - command: | - # Clone RVM at pinned tag and verify the commit SHA matches the - # published tag before running its install script. - RVM_VERSION="1.29.12" - RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81" - git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm - RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)" - if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then - echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2 - exit 1 - fi - - # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs) - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB - - # Install RVM from the verified checkout. The install script - # sources `scripts/functions/installer` using paths relative to - # its own working directory, so it must be run from /tmp/rvm. - (cd /tmp/rvm && ./install --path "$HOME/.rvm") - source "$HOME/.rvm/scripts/rvm" - - # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature) - rvm install 3.2.2 - rvm use 3.2.2 --default - - # Install latest Bundler - gem install bundler - - - run: - name: Run Ruby tests - command: | - source $HOME/.rvm/scripts/rvm - cd tests/pass_through_tests/ruby_passthrough_tests - bundle install - bundle exec rspec - no_output_timeout: 30m # Install Node.js directly from nodejs.org with SHA256 verification, # instead of piping NodeSource's setup_24.x apt-repo installer into # sudo bash (which runs a mutable upstream script unattended). diff --git a/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py b/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py index 1b198623381..2346a5ee047 100644 --- a/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py +++ b/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py @@ -94,11 +94,13 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): langfuse_public_key=None, langfuse_secret=None, langfuse_host=None, + langfuse_environment=None, allow_env_credentials=True, ): captured["langfuse_public_key"] = langfuse_public_key captured["langfuse_secret"] = langfuse_secret captured["langfuse_host"] = langfuse_host + captured["langfuse_environment"] = langfuse_environment captured["allow_env_credentials"] = allow_env_credentials class FakeDynamicLoggingCache: @@ -117,6 +119,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): "langfuse_public_key": "dynamic-public", "langfuse_secret_key": "dynamic-secret", "langfuse_host": "https://langfuse.example", + "langfuse_environment": "dynamic-environment", }, in_memory_dynamic_logger_cache=FakeDynamicLoggingCache(), ) @@ -124,6 +127,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): assert captured["langfuse_public_key"] == "dynamic-public" assert captured["langfuse_secret"] == "dynamic-secret" assert captured["langfuse_host"] == "https://langfuse.example" + assert captured["langfuse_environment"] == "dynamic-environment" assert captured["allow_env_credentials"] is False assert captured["cached_service_name"] == "langfuse" assert captured["cached_logging_obj"] is logger diff --git a/tests/logging_callback_tests/test_langfuse_unit_tests.py b/tests/logging_callback_tests/test_langfuse_unit_tests.py index 1c25b169243..405b6e9e48e 100644 --- a/tests/logging_callback_tests/test_langfuse_unit_tests.py +++ b/tests/logging_callback_tests/test_langfuse_unit_tests.py @@ -123,16 +123,12 @@ def test_get_langfuse_logger_for_request_with_dynamic_params( assert result.secret_key == "test_secret" assert result.langfuse_host == "https://test.langfuse.com" - # Check if the logger is cached - cached_logger = dynamic_logging_cache.get_cache( - credentials={ - "langfuse_public_key": "test_public_key", - "langfuse_secret": "test_secret", - "langfuse_host": "https://test.langfuse.com", - }, - service_name="langfuse", + logger_for_identical_repeat_request = LangFuseHandler.get_langfuse_logger_for_request( + standard_callback_dynamic_params=standard_params, + in_memory_dynamic_logger_cache=dynamic_logging_cache, + globalLangfuseLogger=globalLangfuseLogger, ) - assert cached_logger is result + assert logger_for_identical_repeat_request is result @pytest.mark.parametrize("globalLangfuseLogger", [None, global_langfuse_logger]) diff --git a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile b/tests/pass_through_tests/ruby_passthrough_tests/Gemfile deleted file mode 100644 index 56860496b2b..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile +++ /dev/null @@ -1,4 +0,0 @@ -source 'https://rubygems.org' - -gem 'rspec' -gem 'ruby-openai' \ No newline at end of file diff --git a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock b/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock deleted file mode 100644 index 2072798ccfc..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock +++ /dev/null @@ -1,42 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - base64 (0.2.0) - diff-lcs (1.6.0) - event_stream_parser (1.0.0) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) - faraday-multipart (1.1.0) - multipart-post (~> 2.0) - faraday-net_http (3.0.2) - multipart-post (2.4.1) - rspec (3.13.0) - rspec-core (~> 3.13.0) - rspec-expectations (~> 3.13.0) - rspec-mocks (~> 3.13.0) - rspec-core (3.13.3) - rspec-support (~> 3.13.0) - rspec-expectations (3.13.3) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-mocks (3.13.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-support (3.13.2) - ruby-openai (7.4.0) - event_stream_parser (>= 0.3.0, < 2.0.0) - faraday (>= 1) - faraday-multipart (>= 1) - ruby2_keywords (0.0.5) - -PLATFORMS - ruby - -DEPENDENCIES - rspec - ruby-openai - -BUNDLED WITH - 2.6.5 diff --git a/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb b/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb deleted file mode 100644 index 5a4dc0395f8..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -require 'openai' -require 'rspec' - -RSpec.describe 'OpenAI Assistants Passthrough' do - let(:client) do - OpenAI::Client.new( - access_token: "sk-1234", - uri_base: "http://0.0.0.0:4000/openai", - request_timeout: 600 - ) - end - - - it 'performs basic assistant operations' do - assistant = client.assistants.create( - parameters: { - name: "Math Tutor", - instructions: "You are a personal math tutor. Write and run code to answer math questions.", - tools: [{ type: "code_interpreter" }], - model: "gpt-4o" - } - ) - expect(assistant).to include('id') - expect(assistant['name']).to eq("Math Tutor") - - assistants_list = client.assistants.list - expect(assistants_list['data']).to be_an(Array) - expect(assistants_list['data']).to include(include('id' => assistant['id'])) - - retrieved_assistant = client.assistants.retrieve(id: assistant['id']) - expect(retrieved_assistant).to eq(assistant) - - deleted_assistant = client.assistants.delete(id: assistant['id']) - expect(deleted_assistant['deleted']).to be true - expect(deleted_assistant['id']).to eq(assistant['id']) - end - - it 'performs streaming assistant operations' do - puts "\n=== Starting Streaming Assistant Test ===" - - assistant = client.assistants.create( - parameters: { - name: "Math Tutor", - instructions: "You are a personal math tutor. Write and run code to answer math questions.", - tools: [{ type: "code_interpreter" }], - model: "gpt-4o" - } - ) - puts "Created assistant: #{assistant['id']}" - expect(assistant).to include('id') - - thread = client.threads.create - puts "Created thread: #{thread['id']}" - expect(thread).to include('id') - - message = client.messages.create( - thread_id: thread['id'], - parameters: { - role: "user", - content: "I need to solve the equation `3x + 11 = 14`. Can you help me?" - } - ) - puts "Created message: #{message['id']}" - puts "User question: #{message['content']}" - expect(message).to include('id') - expect(message['role']).to eq('user') - - puts "\nStarting streaming response:" - puts "------------------------" - run = client.runs.create( - thread_id: thread['id'], - parameters: { - assistant_id: assistant['id'], - max_prompt_tokens: 256, - max_completion_tokens: 16, - stream: proc do |chunk, _bytesize| - puts "Received chunk: #{chunk.inspect}" # Debug: Print raw chunk - if chunk["object"] == "thread.message.delta" - content = chunk.dig("delta", "content") - puts "Content: #{content.inspect}" # Debug: Print content structure - if content && content[0] && content[0]["text"] - print content[0]["text"]["value"] - $stdout.flush # Ensure output is printed immediately - end - end - end - } - ) - puts "\n------------------------" - puts "Run completed: #{run['id']}" - expect(run).not_to be_nil - ensure - client.assistants.delete(id: assistant['id']) if assistant && assistant['id'] - client.threads.delete(id: thread['id']) if thread && thread['id'] - end -end \ No newline at end of file diff --git a/tests/pass_through_tests/test_openai_assistants_passthrough.py b/tests/pass_through_tests/test_openai_assistants_passthrough.py index 28568005fd6..78833e85fa9 100644 --- a/tests/pass_through_tests/test_openai_assistants_passthrough.py +++ b/tests/pass_through_tests/test_openai_assistants_passthrough.py @@ -1,7 +1,4 @@ -import pytest import openai -import aiohttp -import asyncio import tempfile from typing_extensions import override from openai import AssistantEventHandler @@ -30,22 +27,6 @@ def test_pass_through_file_operations(): print("file deleted", delete_file) -def test_openai_assistants_e2e_operations(): - assistant = client.beta.assistants.create( - name="Math Tutor", - instructions="You are a personal math tutor. Write and run code to answer math questions.", - tools=[{"type": "code_interpreter"}], - model="gpt-4o", - ) - print("assistant created", assistant) - - get_assistant = client.beta.assistants.retrieve(assistant.id) - print(get_assistant) - - delete_assistant = client.beta.assistants.delete(assistant.id) - print(delete_assistant) - - class EventHandler(AssistantEventHandler): @override def on_text_created(self, text) -> None: @@ -69,39 +50,6 @@ class EventHandler(AssistantEventHandler): print(f"\n{output.logs}", flush=True) -def test_openai_assistants_e2e_operations_stream(): - - assistant = client.beta.assistants.create( - name="Math Tutor", - instructions="You are a personal math tutor. Write and run code to answer math questions.", - tools=[{"type": "code_interpreter"}], - model="gpt-4o", - ) - print("assistant created", assistant) - - thread = client.beta.threads.create() - print("thread created", thread) - - message = client.beta.threads.messages.create( - thread_id=thread.id, - role="user", - content="I need to solve the equation `3x + 11 = 14`. Can you help me?", - ) - print("message created", message) - - # Then, we use the `stream` SDK helper - # with the `EventHandler` class to create the Run - # and stream the response. - - with client.beta.threads.runs.stream( - thread_id=thread.id, - assistant_id=assistant.id, - instructions="Please address the user as Jane Doe. The user has a premium account.", - event_handler=EventHandler(), - ) as stream: - stream.until_done() - - def test_azure_openai_assistants_e2e_operations_stream(): from openai import AzureOpenAI diff --git a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py index e70f2cf4430..d7add68753b 100644 --- a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py +++ b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py @@ -1,7 +1,5 @@ import json import os -from datetime import datetime -from unittest.mock import AsyncMock, Mock, patch, MagicMock from typing import Optional from fastapi import Request import pytest @@ -32,16 +30,16 @@ class TestCustomLogger(CustomLogger): @pytest.mark.asyncio -async def test_assistants_passthrough_logging(): +async def test_untracked_openai_route_passthrough_logging(): + """Keep this on a route `_is_supported_openai_endpoint` does not claim, or the + OpenAI-specific handler takes over and the generic payload stops being exercised.""" test_custom_logger = TestCustomLogger() litellm._async_success_callback = [test_custom_logger] - TARGET_URL = "https://api.openai.com/v1/assistants" + TARGET_URL = "https://api.openai.com/v1/moderations" REQUEST_BODY = { - "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", - "name": "Math Tutor", - "tools": [{"type": "code_interpreter"}], - "model": "gpt-4.1-mini", + "model": "omni-moderation-latest", + "input": "I want to bake a cake for my friend's birthday.", } TARGET_METHOD = "POST" @@ -50,7 +48,7 @@ async def test_assistants_passthrough_logging(): scope={ "type": "http", "method": TARGET_METHOD, - "path": "/v1/assistants", + "path": "/v1/moderations", "query_string": b"", "headers": [ (b"content-type", b"application/json"), @@ -58,7 +56,6 @@ async def test_assistants_passthrough_logging(): b"authorization", f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(), ), - (b"openai-beta", b"assistants=v2"), ], }, ), @@ -66,7 +63,6 @@ async def test_assistants_passthrough_logging(): custom_headers={ "Content-Type": "application/json", "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}", - "OpenAI-Beta": "assistants=v2", }, user_api_key_dict=UserAPIKeyAuth( api_key="test", @@ -83,6 +79,8 @@ async def test_assistants_passthrough_logging(): print("result status code", result.status_code) print("result content", result.body) + assert result.status_code == 200 + await asyncio.sleep(1) assert test_custom_logger.logged_kwargs is not None @@ -92,79 +90,8 @@ async def test_assistants_passthrough_logging(): assert passthrough_logging_payload is not None assert passthrough_logging_payload["url"] == TARGET_URL assert passthrough_logging_payload["request_body"] == REQUEST_BODY - - # assert that the response body content matches the response body content - client_facing_response_body = json.loads(result.body) - assert passthrough_logging_payload["response_body"] == client_facing_response_body - - # assert that the request method is correct assert passthrough_logging_payload["request_method"] == TARGET_METHOD - -@pytest.mark.asyncio -async def test_threads_passthrough_logging(): - test_custom_logger = TestCustomLogger() - litellm._async_success_callback = [test_custom_logger] - - TARGET_URL = "https://api.openai.com/v1/threads" - REQUEST_BODY = {} - TARGET_METHOD = "POST" - - result = await pass_through_request( - request=Request( - scope={ - "type": "http", - "method": TARGET_METHOD, - "path": "/v1/threads", - "query_string": b"", - "headers": [ - (b"content-type", b"application/json"), - ( - b"authorization", - f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(), - ), - (b"openai-beta", b"assistants=v2"), - ], - }, - ), - target=TARGET_URL, - custom_headers={ - "Content-Type": "application/json", - "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}", - "OpenAI-Beta": "assistants=v2", - }, - user_api_key_dict=UserAPIKeyAuth( - api_key="test", - user_id="test", - team_id="test", - end_user_id="test", - ), - custom_body=REQUEST_BODY, - forward_headers=False, - merge_query_params=False, - ) - - print("got result", result) - print("result status code", result.status_code) - print("result content", result.body) - - await asyncio.sleep(1) - - assert test_custom_logger.logged_kwargs is not None - passthrough_logging_payload = test_custom_logger.logged_kwargs[ - "passthrough_logging_payload" - ] - assert passthrough_logging_payload is not None - - # Fix for TypedDict access errors - assert passthrough_logging_payload.get("url") == TARGET_URL - assert passthrough_logging_payload.get("request_body") == REQUEST_BODY - - # Fix for json.loads error with potential memoryview - response_body = result.body - client_facing_response_body = json.loads(response_body) - - assert ( - passthrough_logging_payload.get("response_body") == client_facing_response_body - ) - assert passthrough_logging_payload.get("request_method") == TARGET_METHOD + client_facing_response_body = json.loads(result.body) + assert client_facing_response_body["results"] + assert passthrough_logging_payload["response_body"] == client_facing_response_body From 21092d633b211aad9bebe87a1a1192435efc7169 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 23:43:38 -0700 Subject: [PATCH 02/22] test(opik): stop the batching test racing its own 5-second flush timer test_opik_logging_http_request asserted "nothing has been POSTed yet" roughly one second into a window governed by OpikLogger's 5-second periodic flush. On a loaded CI worker the five preceding acompletion calls eat that budget, the periodic flush fires, and the assertion flips. Reproduced with no product changes at all: letting 5.5 seconds pass before the assertion drains the queue and sets mock_post.called, which is exactly the failure CircleCI reports. The test now pins flush_interval past anything the test can reach, so the two batching assertions measure batching instead of wall clock, and drives the flush path explicitly at the end rather than sleeping the interval. That last phase used to be near-vacuous, since the size-triggered flush had already emptied the queue. Assertions now match only calls to Opik's own /traces/batch and /spans/batch. get_async_httpx_client caches one client per special provider, so the mock is process-wide and any other logging callback's POST would otherwise count. Dropped the teardown that closed that shared client, which broke every later test in the same worker that logs through it, and the try/except that turned assertion failures into a pytest.fail with no traceback. Mutation checked: flushing on every event and never flushing on size both fail the test. --- tests/local_testing/test_opik.py | 98 +++++++++++++++----------------- 1 file changed, 45 insertions(+), 53 deletions(-) diff --git a/tests/local_testing/test_opik.py b/tests/local_testing/test_opik.py index 8be4b796360..2f6b15e1f27 100644 --- a/tests/local_testing/test_opik.py +++ b/tests/local_testing/test_opik.py @@ -16,6 +16,8 @@ verbose_logger.setLevel(logging.DEBUG) litellm.set_verbose = True import time +INTERVAL_TOO_LONG_TO_FIRE_DURING_THIS_TEST = 3600 + @pytest.mark.asyncio async def test_opik_logging_http_request(): @@ -23,70 +25,60 @@ async def test_opik_logging_http_request(): - Test that HTTP requests are made to Opik - Traces and spans are batched correctly """ - try: - from litellm.integrations.opik.opik import OpikLogger + from litellm.integrations.opik.opik import OpikLogger - os.environ["OPIK_URL_OVERRIDE"] = "https://fake.comet.com/opik/api" - os.environ["OPIK_API_KEY"] = "anything" - os.environ["OPIK_WORKSPACE"] = "anything" + os.environ["OPIK_URL_OVERRIDE"] = "https://fake.comet.com/opik/api" + os.environ["OPIK_API_KEY"] = "anything" + os.environ["OPIK_WORKSPACE"] = "anything" - # Initialize OpikLogger - test_opik_logger = OpikLogger() + test_opik_logger = OpikLogger() + test_opik_logger.flush_interval = INTERVAL_TOO_LONG_TO_FIRE_DURING_THIS_TEST + test_opik_logger.batch_size = 12 - litellm.callbacks = [test_opik_logger] - test_opik_logger.batch_size = 12 - litellm.set_verbose = True + litellm.callbacks = [test_opik_logger] - # Create a mock for the async_client's post method - mock_post = AsyncMock() - mock_post.return_value.status_code = 202 - mock_post.return_value.text = "Accepted" - test_opik_logger.async_httpx_client.post = mock_post + mock_post = AsyncMock(return_value=Mock(status_code=202, text="Accepted")) + test_opik_logger.async_httpx_client.post = mock_post - # Make multiple calls to ensure we don't hit the batch size - for _ in range(5): - response = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Test message"}], - max_tokens=10, - temperature=0.2, - mock_response="This is a mock response", - ) - await asyncio.sleep(1) + def opik_batch_calls(): + return [ + call + for call in mock_post.call_args_list + if "/traces/batch" in str(call) or "/spans/batch" in str(call) + ] - # Check batching of events and that the queue contains 5 trace events and 5 span events - assert ( - mock_post.called == False - ), "HTTP request was made but events should have been batched" - assert len(test_opik_logger.log_queue) == 10 + for _ in range(5): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Test message"}], + max_tokens=10, + temperature=0.2, + mock_response="This is a mock response", + ) + await asyncio.sleep(1) - # Now make calls to exceed the batch size - for _ in range(3): - response = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Test message"}], - max_tokens=10, - temperature=0.2, - mock_response="This is a mock response", - ) + assert opik_batch_calls() == [], "events below batch_size must stay queued" + assert len(test_opik_logger.log_queue) == 10 - # Wait a short time for any asynchronous operations to complete - await asyncio.sleep(1) + for _ in range(3): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Test message"}], + max_tokens=10, + temperature=0.2, + mock_response="This is a mock response", + ) + await asyncio.sleep(1) - # Check that the queue was flushed after exceeding batch size - assert len(test_opik_logger.log_queue) < test_opik_logger.batch_size + assert opik_batch_calls(), "crossing batch_size must flush the queue" + events_left_over_after_the_size_triggered_flush = len(test_opik_logger.log_queue) + assert 0 < events_left_over_after_the_size_triggered_flush < test_opik_logger.batch_size - # Check that the data has been sent when it goes above the flush interval - await asyncio.sleep(test_opik_logger.flush_interval) - assert len(test_opik_logger.log_queue) == 0 + calls_before_periodic_flush = len(opik_batch_calls()) + await test_opik_logger.flush_queue() - # Clean up - for cb in litellm.callbacks: - if isinstance(cb, OpikLogger): - await cb.async_httpx_client.client.aclose() - - except Exception as e: - pytest.fail(f"Error occurred: {e}") + assert len(opik_batch_calls()) > calls_before_periodic_flush + assert len(test_opik_logger.log_queue) == 0 def test_sync_opik_logging_http_request(): From 120971dacc66bb8a6962d4e71964b91923c1cf2f Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 23:53:14 -0700 Subject: [PATCH 03/22] chore(lint): ratchet the TQ ceilings down to what these test fixes reached --- test-quality-budget.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index 4a7bc7edff2..747ae07a06b 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -1,6 +1,6 @@ { "TQ001": { - "limit": 744 + "limit": 742 }, "TQ002": { "limit": 742 @@ -12,7 +12,7 @@ "limit": 469 }, "TQ005": { - "limit": 2405 + "limit": 2403 }, "TQ006": { "limit": 34 From 1eedaa3a43e21fcc617529b1a8b75cd3f7d7d981 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 27 Aug 2026 23:58:23 -0700 Subject: [PATCH 04/22] test(passthrough): drop the Azure assistants test, retired upstream on 2026-08-26 Azure now answers the create call with 410 and code assistants_api_deprecated: "The Assistants API has been retired. Follow the migration guide to update your workloads." Microsoft retired it on the same day OpenAI retired theirs, which is why this landed with the OpenAI ones rather than before them. This job runs pytest with -x, so the test was also hiding everything after it in tests/pass_through_tests. test_pass_through_file_operations stays. It only asks /v1/files for purpose="assistants", which still answers 200 on both upload and delete. --- .../test_openai_assistants_passthrough.py | 67 ------------------- 1 file changed, 67 deletions(-) diff --git a/tests/pass_through_tests/test_openai_assistants_passthrough.py b/tests/pass_through_tests/test_openai_assistants_passthrough.py index 78833e85fa9..9afd8b23b2f 100644 --- a/tests/pass_through_tests/test_openai_assistants_passthrough.py +++ b/tests/pass_through_tests/test_openai_assistants_passthrough.py @@ -1,89 +1,22 @@ import openai import tempfile -from typing_extensions import override -from openai import AssistantEventHandler client = openai.OpenAI(base_url="http://0.0.0.0:4000/openai", api_key="sk-1234") def test_pass_through_file_operations(): - # Create a temporary file with tempfile.NamedTemporaryFile( mode="w+", suffix=".txt", delete=False ) as temp_file: temp_file.write("This is a test file for the OpenAI Assistants API.") temp_file.flush() - # create a file file = client.files.create( file=open(temp_file.name, "rb"), purpose="assistants", ) print("file created", file) - # delete the file delete_file = client.files.delete(file.id) print("file deleted", delete_file) - - -class EventHandler(AssistantEventHandler): - @override - def on_text_created(self, text) -> None: - print(f"\nassistant > ", end="", flush=True) - - @override - def on_text_delta(self, delta, snapshot): - print(delta.value, end="", flush=True) - - def on_tool_call_created(self, tool_call): - print(f"\nassistant > {tool_call.type}\n", flush=True) - - def on_tool_call_delta(self, delta, snapshot): - if delta.type == "code_interpreter": - if delta.code_interpreter.input: - print(delta.code_interpreter.input, end="", flush=True) - if delta.code_interpreter.outputs: - print(f"\n\noutput >", flush=True) - for output in delta.code_interpreter.outputs: - if output.type == "logs": - print(f"\n{output.logs}", flush=True) - - -def test_azure_openai_assistants_e2e_operations_stream(): - from openai import AzureOpenAI - - client = AzureOpenAI( - base_url="http://0.0.0.0:4000/azure-config-passthrough/openai", - api_key="sk-1234", - api_version="2025-01-01-preview", - ) - assistant = client.beta.assistants.create( - name="Math Tutor", - instructions="You are a personal math tutor. Write and run code to answer math questions.", - tools=[{"type": "code_interpreter"}], - model="gpt-4o", - ) - print("assistant created", assistant) - - thread = client.beta.threads.create() - print("thread created", thread) - - message = client.beta.threads.messages.create( - thread_id=thread.id, - role="user", - content="I need to solve the equation `3x + 11 = 14`. Can you help me?", - ) - print("message created", message) - - # Then, we use the `stream` SDK helper - # with the `EventHandler` class to create the Run - # and stream the response. - - with client.beta.threads.runs.stream( - thread_id=thread.id, - assistant_id=assistant.id, - instructions="Please address the user as Jane Doe. The user has a premium account.", - event_handler=EventHandler(), - ) as stream: - stream.until_done() From 0c5c96dbf6f258f7f2fda9ea9198becaf85c9cf1 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 00:01:51 -0700 Subject: [PATCH 05/22] test(e2e): unskip four tests whose blockers no longer hold /v1/batches now rejects a missing input_file_id with a 400 through raise_if_required_body_param_missing, so the contract negative that was skipped for "500s instead of 400" passes as written. Verified against a live proxy. The three Datadog MCP tests were skipped because each one sent a `telemetry` argument that search_datadog_logs rejects with "unexpected additional properties". That argument was never a documented Datadog parameter and no assertion reads it, so it is dropped and the tests run again unchanged otherwise. --- .../test_files_batches_contract_e2e.py | 3 --- tests/e2e/mcp/test_mcp_datadog_e2e.py | 13 ------------- tests/e2e/mcp/test_mcp_guardrail_e2e.py | 11 ----------- tests/e2e/mcp/test_mcp_key_access_e2e.py | 11 ----------- 4 files changed, 38 deletions(-) diff --git a/tests/e2e/llm_translation/test_files_batches_contract_e2e.py b/tests/e2e/llm_translation/test_files_batches_contract_e2e.py index b1166891164..5627fa1c0bf 100644 --- a/tests/e2e/llm_translation/test_files_batches_contract_e2e.py +++ b/tests/e2e/llm_translation/test_files_batches_contract_e2e.py @@ -46,9 +46,6 @@ class TestFilesBatchesContract: case other: pytest.fail(f"upload without purpose expected 4xx, got {other!r}") - @pytest.mark.skip( - reason="stage red: product gap, /v1/batches 500s (acreate_batch TypeError) on missing input_file_id instead of 400" - ) @pytest.mark.covers("llm.batches.openai.input_validation.nonstream.works") def test_create_batch_missing_input_file_id_returns_error( self, proxy: ProxyClient, resources: ResourceManager diff --git a/tests/e2e/mcp/test_mcp_datadog_e2e.py b/tests/e2e/mcp/test_mcp_datadog_e2e.py index 138f654272d..031fbf6d936 100644 --- a/tests/e2e/mcp/test_mcp_datadog_e2e.py +++ b/tests/e2e/mcp/test_mcp_datadog_e2e.py @@ -49,16 +49,6 @@ def _seed_completion(proxy: ProxyClient, *, key: str, marker: str) -> None: class TestDatadogMcpRoundTrip: - @pytest.mark.skip( - reason=( - "LIT-5052: this test sends a `telemetry` argument that Datadog's " - "search_datadog_logs tool now rejects, so every tool call fails validation with " - "'unexpected additional properties [\"telemetry\"]' before the round-trip " - "assertion is reached. `telemetry` was never a documented Datadog parameter; the " - "test relied on the server ignoring unknown properties. Unskip once the argument " - "is dropped." - ) - ) @pytest.mark.covers("mcp.list_tools.api_key.succeeds", "mcp.call_tool.api_key.succeeds") def test_search_logs_finds_seeded_completion( self, @@ -98,9 +88,6 @@ class TestDatadogMcpRoundTrip: "from": DD_SEARCH_FROM, "to": "now", "max_tokens": 5000, - "telemetry": { - "intent": "e2e assert seeded litellm completion log is searchable via MCP" - }, }, ) assert call.is_error is not True, f"search_datadog_logs errored: {call}" diff --git a/tests/e2e/mcp/test_mcp_guardrail_e2e.py b/tests/e2e/mcp/test_mcp_guardrail_e2e.py index 60a349ddc5e..92c632cb316 100644 --- a/tests/e2e/mcp/test_mcp_guardrail_e2e.py +++ b/tests/e2e/mcp/test_mcp_guardrail_e2e.py @@ -78,16 +78,6 @@ def _search_on_synced_pod( class TestMcpToolCallGuardrail: - @pytest.mark.skip( - reason=( - "LIT-5052: the control call sends a `telemetry` argument that Datadog's " - "search_datadog_logs tool now rejects, so the clean-argument half of this test " - "errors with 'unexpected additional properties [\"telemetry\"]' and the guardrail " - "block it exists to prove is never exercised. `telemetry` was never a documented " - "Datadog parameter; the test relied on the server ignoring unknown properties. " - "Unskip once the argument is dropped." - ) - ) @pytest.mark.covers( "guardrail.litellm_content_filter.pre_mcp_call.blocks", exercised_on=["mcp_operations"], @@ -118,7 +108,6 @@ class TestMcpToolCallGuardrail: "from": DD_SEARCH_FROM, "to": "now", "max_tokens": 500, - "telemetry": {"intent": "e2e mcp guardrail check"}, } return client.call_tool(key, server_id=server_id, name=tool_name, arguments=arguments) diff --git a/tests/e2e/mcp/test_mcp_key_access_e2e.py b/tests/e2e/mcp/test_mcp_key_access_e2e.py index 788a0a3f45c..88ab5666084 100644 --- a/tests/e2e/mcp/test_mcp_key_access_e2e.py +++ b/tests/e2e/mcp/test_mcp_key_access_e2e.py @@ -51,16 +51,6 @@ class TestMcpKeyWithoutAccessIsDenied: f"boundary: {denied_tools}" ) - @pytest.mark.skip( - reason=( - "LIT-5052: the control call proving a granted key CAN invoke the tool sends a " - "`telemetry` argument that Datadog's search_datadog_logs tool now rejects, so it " - "errors with 'unexpected additional properties [\"telemetry\"]' and the denial " - "assertion is never reached. `telemetry` was never a documented Datadog " - "parameter; the test relied on the server ignoring unknown properties. Unskip " - "once the argument is dropped." - ) - ) @pytest.mark.covers("mcp.call_tool.api_key.denied_without_permission") def test_call_tool_denied_without_permission( self, @@ -80,7 +70,6 @@ class TestMcpKeyWithoutAccessIsDenied: "from": DD_SEARCH_FROM, "to": "now", "max_tokens": 1000, - "telemetry": {"intent": "e2e control call proving granted key can invoke Datadog MCP"}, } permitted_call = client.await_call_tool( permitted_key, server_id=server_id, name=tool_name, arguments=search_args From d8679508d4c1d1992923e75295df62188e4ed6b9 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 00:15:02 -0700 Subject: [PATCH 06/22] test(e2e): measure the select popup after it settles instead of mid-flight Both anchoring tests read the trigger's box before the click and the popup's box the instant it turns visible. Base UI places the popup asynchronously and opening it can shift the trigger, so both boxes could be sampled before the layout settled. The run on 1eedaa3a43 missed by 4.2px (expected >= 446.015, got 441.799) on a tree with no UI changes at all, having passed on 21092d633b, which differs only in a deleted python test and a budget json. Each assertion now re-reads both boxes under expect.poll. The conditions themselves are unchanged: the popup must sit at or below the trigger's bottom edge in the first test and must not overlap it in the second. Polling cannot mask a genuinely misplaced popup, since one that never lands correctly still fails when the poll times out. --- .../autoRouterTemplateSelect.spec.ts | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts index 98bd1b84f11..1d080ec82b8 100644 --- a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts @@ -1,4 +1,4 @@ -import { expect, test, type Page as PlaywrightPage } from "@playwright/test"; +import { expect, test, type Locator, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; @@ -17,43 +17,49 @@ async function openTemplateSelect(page: PlaywrightPage) { return trigger; } +function pollPixelsBelowTrigger(trigger: Locator, popup: Locator) { + return expect.poll(async () => { + const triggerBox = await trigger.boundingBox(); + const popupBox = await popup.boundingBox(); + if (!triggerBox || !popupBox) return null; + return popupBox.y - (triggerBox.y + triggerBox.height); + }); +} + +function pollPopupOverlapsTrigger(trigger: Locator, popup: Locator) { + return expect.poll(async () => { + const triggerBox = await trigger.boundingBox(); + const popupBox = await popup.boundingBox(); + if (!triggerBox || !popupBox) return null; + return popupBox.y < triggerBox.y + triggerBox.height && popupBox.y + popupBox.height > triggerBox.y; + }); +} + test.describe("Auto Router template select anchoring", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); test("opens the options below the trigger rather than over it", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); const trigger = await openTemplateSelect(page); - const triggerBox = await trigger.boundingBox(); await trigger.click(); const popup = page.locator('[data-slot="select-content"]'); await expect(popup).toBeVisible(); - const popupBox = await popup.boundingBox(); - - expect(triggerBox).not.toBeNull(); - expect(popupBox).not.toBeNull(); // Item-aligned mode reports "none" and puts the active item over the trigger. await expect(popup).toHaveAttribute("data-side", "bottom"); - expect(popupBox!.y).toBeGreaterThanOrEqual(triggerBox!.y + triggerBox!.height); + await pollPixelsBelowTrigger(trigger, popup).toBeGreaterThanOrEqual(0); }); test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 560 }); const trigger = await openTemplateSelect(page); await trigger.scrollIntoViewIfNeeded(); - const triggerBox = await trigger.boundingBox(); await trigger.click(); const popup = page.locator('[data-slot="select-content"]'); await expect(popup).toBeVisible(); - const popupBox = await popup.boundingBox(); - expect(triggerBox).not.toBeNull(); - expect(popupBox).not.toBeNull(); - - const overlaps = - popupBox!.y < triggerBox!.y + triggerBox!.height && popupBox!.y + popupBox!.height > triggerBox!.y; - expect(overlaps).toBe(false); + await pollPopupOverlapsTrigger(trigger, popup).toBe(false); }); }); From e8b9f3675bb24260cb447f5c11b18532700f66e8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 00:25:53 -0700 Subject: [PATCH 07/22] test(batch): make the upstream-failure tolerance actually reachable batch_completion collects per-request failures into its result list rather than raising them; its own source says "return exceptions if any". So the test's `except Timeout` and `except litellm.InternalServerError` arms could never fire for the case they were written for. An upstream 500 instead reached `response.choices`, raised AttributeError on the exception object, and fell through to the bare `except Exception` that calls pytest.fail. That is what CircleCI hit. The tolerance now reads the returned values, which is where the failures actually are. The same two exception types are tolerated as before, nothing broader. Checked against four injected outcomes: three InternalServerErrors pass, three Timeouts pass, an AuthenticationError fails, and a response whose content is None fails. So it is not tolerating its way to a vacuous green. --- tests/local_testing/test_batch_completions.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/local_testing/test_batch_completions.py b/tests/local_testing/test_batch_completions.py index d3296988e8c..0f9b628823a 100644 --- a/tests/local_testing/test_batch_completions.py +++ b/tests/local_testing/test_batch_completions.py @@ -19,32 +19,32 @@ from litellm import ( # litellm.set_verbose=True +TOLERATED_UPSTREAM_FAILURES = (Timeout, litellm.InternalServerError) + + def test_batch_completions(): messages = [[{"role": "user", "content": "write a short poem"}] for _ in range(3)] model = "gpt-3.5-turbo" litellm.set_verbose = True - try: - result = batch_completion( - model=model, - messages=messages, - max_tokens=10, - temperature=0.2, - request_timeout=1, - ) - print(result) - print(len(result)) - assert len(result) == 3 - for response in result: - assert response.choices[0].message.content is not None - except Timeout as e: - print(f"IN TIMEOUT") - pass - except litellm.InternalServerError as e: - print(f"IN INTERNAL SERVER ERROR") - pass - except Exception as e: - pytest.fail(f"An error occurred: {e}") + result = batch_completion( + model=model, + messages=messages, + max_tokens=10, + temperature=0.2, + request_timeout=1, + ) + print(result) + + assert len(result) == 3 + + for response in result: + if isinstance(response, TOLERATED_UPSTREAM_FAILURES): + continue + assert not isinstance( + response, Exception + ), f"batch_completion returned {type(response).__name__}: {response}" + assert response.choices[0].message.content is not None # test_batch_completions() From 4cc6119d0898275cb3becf5092b0a6ab55439c31 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 00:27:12 -0700 Subject: [PATCH 08/22] chore(lint): ratchet the TQ ceilings for the batch completions fix --- test-quality-budget.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index 747ae07a06b..cab7a027e93 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -1,6 +1,6 @@ { "TQ001": { - "limit": 742 + "limit": 739 }, "TQ002": { "limit": 742 @@ -12,7 +12,7 @@ "limit": 469 }, "TQ005": { - "limit": 2403 + "limit": 2401 }, "TQ006": { "limit": 34 From 1cce589aa008609cdd6e1d3ebf6db37c37c27d51 Mon Sep 17 00:00:00 2001 From: Fazeel Usmani Date: Wed, 12 Aug 2026 20:45:24 +0530 Subject: [PATCH 09/22] fix(token-counter): count Anthropic native image content blocks `_count_content_list` accepted text, image_url, tool_use, tool_result, thinking and tool_reference, and raised on anything else, so an Anthropic-native `{"type": "image", "source": {...}}` block aborted the whole count. That is the documented Anthropic image format and exactly what /v1/messages receives. Three user-visible effects. /v1/messages/count_tokens and /utils/token_counter return 500, and the router's context-window pre-call check swallows the ValueError and returns every deployment unfiltered, so an oversized prompt carrying an image is dispatched to the provider instead of being rejected locally with a 400. Prices the block through the existing image path: a base64 source becomes a data URI, a url source passes through, and a file source falls back to the default image token count. Blocks nested inside tool_result.content are covered too, because _count_anthropic_content recurses back into _count_content_list. Fixes #36604 --- litellm/litellm_core_utils/token_counter.py | 28 +++- .../litellm_core_utils/test_token_counter.py | 120 ++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 858b078d626..ccfce0e4133 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -646,6 +646,24 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type: return expected_cls +def _anthropic_image_source_data(source: Mapping[str, str]) -> str: + """ + Resolve an Anthropic image `source` to the data string `calculate_img_tokens` prices. + + Returns "" for a `file` source, whose bytes the proxy cannot resolve locally. + """ + source_type: Final = source.get("type") + if source_type == "base64": + data: Final = source.get("data") + if not data: + return "" + media_type: Final = source.get("media_type") or "image/png" + return f"data:{media_type};base64,{data}" + if source_type == "url": + return source.get("url") or "" + return "" + + def _count_anthropic_content( content: Mapping[str, Any], count_function: TokenCounterFunction, @@ -714,6 +732,13 @@ def _count_content_list( elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) + elif c["type"] == "image": + source = c.get("source") + num_tokens += calculate_img_tokens( + data=_anthropic_image_source_data(source) if isinstance(source, dict) else "", + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( c, @@ -742,7 +767,8 @@ def _count_content_list( content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__ raise ValueError( f"Invalid content item type: {content_type}. " - f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)." + f"Expected str or dict with 'type' field " + f"(text, image_url, image, tool_use, tool_result, thinking, tool_reference)." ) return num_tokens except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index a2590dbca2d..701a1accd5f 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1160,3 +1160,123 @@ def test_count_content_list_rejects_unknown_type(): message = str(exc_info.value) assert "Invalid content item type: totally_unknown_block" in message assert "tool_reference" in message + + +@pytest.mark.parametrize( + "source", + [ + {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}, + {"type": "url", "url": "https://example.com/image.png"}, + {"type": "file", "file_id": "file-abc123"}, + ], + ids=["base64", "url", "file"], +) +def test_token_counter_with_anthropic_image_block(source): + """ + Anthropic-native `image` blocks must NOT raise, for every source variant. + + Before this fix `_count_content_list` raised + `Invalid content item type: image`. That 500s /v1/messages/count_tokens and + /utils/token_counter, and it makes the router's context-window pre-call + check swallow the error and return every deployment unfiltered, so an + oversized prompt carrying an image is dispatched upstream instead of being + rejected locally. + """ + from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image", "source": source}, + ], + } + ] + + tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + use_default_image_token_count=True, + ) + assert tokens > DEFAULT_IMAGE_TOKEN_COUNT, ( + f"Expected the image block to contribute tokens, got {tokens}" + ) + + +def test_anthropic_image_block_matches_equivalent_image_url(): + """ + An Anthropic `image` block must price identically to the OpenAI `image_url` + block carrying the same bytes, so the count does not depend on which + endpoint shape the caller used. + """ + anthropic_messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + } + ], + } + ] + openai_messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + } + ], + } + ] + + anthropic_tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=anthropic_messages + ) + openai_tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=openai_messages + ) + assert anthropic_tokens == openai_tokens + + +def test_anthropic_image_block_nested_in_tool_result(): + """ + An `image` block nested inside a `tool_result.content` list must be counted + too. `_count_anthropic_content` recurses back into `_count_content_list`, so + the nested case failed for the same reason the top-level one did. + """ + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + } + ], + } + ], + } + ] + + tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + use_default_image_token_count=True, + ) + assert tokens > 0 From fe28781dfd0e6012769ab13ac033f14926753633 Mon Sep 17 00:00:00 2001 From: Fazeel Usmani Date: Thu, 13 Aug 2026 13:17:46 +0530 Subject: [PATCH 10/22] fix(token-counter): enhance handling of Anthropic image blocks in token counting --- litellm/litellm_core_utils/token_counter.py | 26 ++++++---- .../litellm_core_utils/test_token_counter.py | 47 +++++++++++++++++++ 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index ccfce0e4133..f101188a692 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,7 +3,7 @@ import base64 import io import struct -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from typing import Any, Final, Literal, cast import tiktoken @@ -25,6 +25,10 @@ from litellm.litellm_core_utils.default_encoding import encoding as default_enco from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client from litellm.types.llms.anthropic import ( + AnthropicContentParamSource, + AnthropicContentParamSourceFileId, + AnthropicContentParamSourceUrl, + AnthropicMessagesImageParam, AnthropicMessagesToolResultParam, AnthropicMessagesToolUseParam, ) @@ -32,7 +36,7 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionNamedToolChoiceParam, ChatCompletionToolParam, - OpenAIMessageContent, + OpenAIMessageContentListBlock, ) from litellm.types.utils import Message, SelectTokenizerResponse @@ -646,20 +650,21 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type: return expected_cls -def _anthropic_image_source_data(source: Mapping[str, str]) -> str: +def _anthropic_image_source_data( + source: AnthropicContentParamSource | AnthropicContentParamSourceUrl | AnthropicContentParamSourceFileId, +) -> str: """ Resolve an Anthropic image `source` to the data string `calculate_img_tokens` prices. Returns "" for a `file` source, whose bytes the proxy cannot resolve locally. """ - source_type: Final = source.get("type") - if source_type == "base64": + if source["type"] == "base64": data: Final = source.get("data") if not data: return "" media_type: Final = source.get("media_type") or "image/png" return f"data:{media_type};base64,{data}" - if source_type == "url": + if source["type"] == "url": return source.get("url") or "" return "" @@ -715,12 +720,16 @@ def _count_anthropic_content( def _count_content_list( count_function: TokenCounterFunction, - content_list: OpenAIMessageContent, + content_list: str | Iterable[OpenAIMessageContentListBlock | AnthropicMessagesImageParam], use_default_image_token_count: bool, default_token_count: int | None, ) -> int: """ Recursively count tokens from a list of content blocks. + + The block union is wider than OpenAI's: the proxy's Anthropic endpoints count + their native blocks through this same helper, so an `image` block is as much + an input here as OpenAI's `image_url`. """ try: num_tokens = 0 @@ -733,9 +742,8 @@ def _count_content_list( image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) elif c["type"] == "image": - source = c.get("source") num_tokens += calculate_img_tokens( - data=_anthropic_image_source_data(source) if isinstance(source, dict) else "", + data=_anthropic_image_source_data(c["source"]), mode="auto", use_default_image_token_count=use_default_image_token_count, ) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 701a1accd5f..e3090d9751b 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1280,3 +1280,50 @@ def test_anthropic_image_block_nested_in_tool_result(): use_default_image_token_count=True, ) assert tokens > 0 + + +def test_anthropic_image_block_with_empty_base64_data(): + """ + A base64 source carrying no bytes must still price as an image rather than + raise: the block is well-formed enough to count, and an empty `data` only + means there is nothing to measure the dimensions from. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + tokens = _count_content_list( + count_function=len, + content_list=[ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}} + ], + use_default_image_token_count=False, + default_token_count=None, + ) + assert tokens > 0 + + +def test_anthropic_image_block_without_source_raises(): + """ + An `image` block with no `source` is malformed, and must fail the same way + the OpenAI `image_url` block with no `url` does - a ValueError the caller + can turn into a 400 - instead of being silently counted as a valid image. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + with pytest.raises(ValueError): + _count_content_list( + count_function=len, + content_list=[{"type": "image"}], + use_default_image_token_count=False, + default_token_count=None, + ) + + # ... and `default_token_count`, the caller's opt-out from raising, still wins. + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "image"}], + use_default_image_token_count=False, + default_token_count=7, + ) + == 7 + ) From 7cd3a27d6007493fdf4819b72ce2e9dbb5b07688 Mon Sep 17 00:00:00 2001 From: Fazeel Usmani Date: Thu, 13 Aug 2026 13:37:53 +0530 Subject: [PATCH 11/22] update test signature for Anthropic image block handling --- tests/test_litellm/litellm_core_utils/test_token_counter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index e3090d9751b..7019577cca0 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1171,7 +1171,7 @@ def test_count_content_list_rejects_unknown_type(): ], ids=["base64", "url", "file"], ) -def test_token_counter_with_anthropic_image_block(source): +def test_token_counter_with_anthropic_image_block(source: dict[str, str]): """ Anthropic-native `image` blocks must NOT raise, for every source variant. From 70ba0bb97399e2ab55bc22a65dd1b4ccba18bf9a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:36:09 -0700 Subject: [PATCH 12/22] fix(proxy): count tools, system, and Anthropic document blocks in the count_tokens fallback --- litellm/litellm_core_utils/token_counter.py | 56 ++++++++++++- litellm/proxy/proxy_server.py | 27 +++++- litellm/types/llms/anthropic.py | 21 ++++- litellm/utils.py | 4 +- .../litellm_core_utils/test_token_counter.py | 84 ++++++++++++++++++- .../proxy/proxy_server/test_routes_utils.py | 45 ++++++++++ 6 files changed, 226 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index f101188a692..98bcedf43fe 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,7 +3,7 @@ import base64 import io import struct -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping, Sequence from typing import Any, Final, Literal, cast import tiktoken @@ -28,12 +28,15 @@ from litellm.types.llms.anthropic import ( AnthropicContentParamSource, AnthropicContentParamSourceFileId, AnthropicContentParamSourceUrl, + AnthropicMessagesDocumentParam, AnthropicMessagesImageParam, + AnthropicMessagesTextParam, AnthropicMessagesToolResultParam, AnthropicMessagesToolUseParam, ) from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionDocumentObject, ChatCompletionNamedToolChoiceParam, ChatCompletionToolParam, OpenAIMessageContentListBlock, @@ -350,7 +353,7 @@ def token_counter( model="", custom_tokenizer: dict | SelectTokenizerResponse | None = None, text: str | list[str] | None = None, - messages: list[AllMessageValues | Message] | None = None, + messages: Sequence[AllMessageValues | Message] | None = None, count_response_tokens: bool | None = False, tools: list[ChatCompletionToolParam] | None = None, tool_choice: ChatCompletionNamedToolChoiceParam | None = None, @@ -669,6 +672,38 @@ def _anthropic_image_source_data( return "" +def _count_document_tokens( + document: ChatCompletionDocumentObject | AnthropicMessagesDocumentParam, + count_function: TokenCounterFunction, + use_default_image_token_count: bool, + default_token_count: int | None, +) -> int: + """ + Count an Anthropic `document` block: its title and context text, plus the source itself. + + Text-bearing sources (`text`, `content`) count their text; opaque ones (`base64`, `url`, + `file`) are priced like an image, since their bytes cannot be tokenized locally. + """ + source: Final = document["source"] + metadata_tokens: Final = sum( + count_function(text) for text in (document.get("title"), document.get("context")) if text + ) + if source["type"] == "text": + return metadata_tokens + count_function(source["data"]) + if source["type"] == "content": + content: Final = source["content"] + if isinstance(content, str): + return metadata_tokens + count_function(content) + return metadata_tokens + _count_content_list( + count_function, content, use_default_image_token_count, default_token_count + ) + return metadata_tokens + calculate_img_tokens( + data=_anthropic_image_source_data(source), + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) + + def _count_anthropic_content( content: Mapping[str, Any], count_function: TokenCounterFunction, @@ -720,7 +755,13 @@ def _count_anthropic_content( def _count_content_list( count_function: TokenCounterFunction, - content_list: str | Iterable[OpenAIMessageContentListBlock | AnthropicMessagesImageParam], + content_list: str + | Iterable[ + OpenAIMessageContentListBlock + | AnthropicMessagesTextParam + | AnthropicMessagesImageParam + | AnthropicMessagesDocumentParam + ], use_default_image_token_count: bool, default_token_count: int | None, ) -> int: @@ -747,6 +788,13 @@ def _count_content_list( mode="auto", use_default_image_token_count=use_default_image_token_count, ) + elif c["type"] == "document": + num_tokens += _count_document_tokens( + c, + count_function, + use_default_image_token_count, + default_token_count, + ) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( c, @@ -776,7 +824,7 @@ def _count_content_list( raise ValueError( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " - f"(text, image_url, image, tool_use, tool_result, thinking, tool_reference)." + f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)." ) return num_tokens except Exception as e: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7ea6fdee6a5..f107eafd283 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -672,7 +672,12 @@ from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseUsageBlock, ) -from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionSystemMessage, + ChatCompletionToolParam, + HttpxBinaryResponseContent, +) from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, @@ -12126,6 +12131,13 @@ async def _try_provider_token_count( return result +def _system_message(system: object) -> ChatCompletionSystemMessage | None: + if not isinstance(system, (str, list)) or not system: + return None + message: Final[ChatCompletionSystemMessage] = {"role": "system", "content": system} + return message + + @router.post( "/utils/token_counter", tags=["llm utils"], @@ -12224,10 +12236,21 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False) _tokenizer_used: Final = litellm.utils._select_tokenizer(model=model_to_use, custom_tokenizer=custom_tokenizer) tokenizer_used: Final = str(_tokenizer_used["type"]) + system_message: Final = _system_message(system) + typed_messages: Final = cast( # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes + Sequence[AllMessageValues] | None, messages + ) + counted_messages: Final = ( + typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) + ) + counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats + list[ChatCompletionToolParam] | None, tools + ) total_tokens: Final = await asyncify(litellm.token_counter)( model=model_to_use, text=prompt, - messages=messages, + messages=counted_messages, + tools=counted_tools, custom_tokenizer=_tokenizer_used, ) return TokenCountResponse( diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 7805dd595a2..b3462203c4b 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -1,4 +1,4 @@ -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from enum import Enum from typing import Any, Final, Literal, TypeAlias @@ -254,6 +254,17 @@ class AnthropicContentParamSourceFileId(TypedDict): file_id: str +class AnthropicContentParamSourceText(TypedDict): + type: ReadOnly[Literal["text"]] + media_type: ReadOnly[Literal["text/plain"]] + data: ReadOnly[str] + + +class AnthropicContentParamSourceContent(TypedDict): + type: ReadOnly[Literal["content"]] + content: ReadOnly[str | Sequence["AnthropicMessagesTextParam | AnthropicMessagesImageParam"]] + + class AnthropicMessagesContainerUploadParam(TypedDict, total=False): type: Required[Literal["container_upload"]] file_id: str @@ -305,7 +316,13 @@ AnthropicCitation = AnthropicCitationPageLocation | AnthropicCitationCharLocatio class AnthropicMessagesDocumentParam(TypedDict, total=False): type: Required[Literal["document"]] - source: Required[AnthropicContentParamSource | AnthropicContentParamSourceFileId | AnthropicContentParamSourceUrl] + source: Required[ + AnthropicContentParamSource + | AnthropicContentParamSourceFileId + | AnthropicContentParamSourceUrl + | AnthropicContentParamSourceText + | AnthropicContentParamSourceContent + ] cache_control: dict | ChatCompletionCachedContent | None title: str context: str diff --git a/litellm/utils.py b/litellm/utils.py index 520c40f67c0..5c5fe7cd97f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2302,7 +2302,7 @@ def token_counter( model="", custom_tokenizer: dict | SelectTokenizerResponse | None = None, text: str | list[str] | None = None, - messages: list | None = None, + messages: Sequence | None = None, count_response_tokens: bool | None = False, tools: list[ChatCompletionToolParam] | None = None, tool_choice: ChatCompletionNamedToolChoiceParam | None = None, @@ -7741,7 +7741,7 @@ def convert_to_dict(message: BaseModel | dict) -> dict: raise TypeError(f"Invalid message type: {type(message)}. Expected dict or Pydantic model.") -def convert_list_message_to_dict(messages: list): +def convert_list_message_to_dict(messages: Sequence): new_messages: Final = [] for message in messages: convert_msg_to_dict = cast(AllMessageValues, convert_to_dict(message)) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 7019577cca0..b8f001240c9 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1282,6 +1282,26 @@ def test_anthropic_image_block_nested_in_tool_result(): assert tokens > 0 +@pytest.mark.parametrize( + ("source", "expected"), + [ + ({"type": "base64", "media_type": "image/jpeg", "data": "/9j/4AAQ"}, "data:image/jpeg;base64,/9j/4AAQ"), + ({"type": "url", "url": "https://example.com/image.png"}, "https://example.com/image.png"), + ({"type": "file", "file_id": "file-abc123"}, ""), + ], + ids=["base64", "url", "file"], +) +def test_anthropic_image_source_resolves_to_what_the_image_pricer_reads(source: dict[str, str], expected: str): + """ + The image pricer reads either a data URI or a fetchable URL: a base64 source keeps its + media type inside the URI, a url source passes through untouched, and a file source has + no bytes the proxy can measure locally. + """ + from litellm.litellm_core_utils.token_counter import _anthropic_image_source_data + + assert _anthropic_image_source_data(source) == expected + + def test_anthropic_image_block_with_empty_base64_data(): """ A base64 source carrying no bytes must still price as an image rather than @@ -1309,7 +1329,7 @@ def test_anthropic_image_block_without_source_raises(): """ from litellm.litellm_core_utils.token_counter import _count_content_list - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Error getting number of tokens from content list"): _count_content_list( count_function=len, content_list=[{"type": "image"}], @@ -1327,3 +1347,65 @@ def test_anthropic_image_block_without_source_raises(): ) == 7 ) + + +def _count_user_content(content: list[dict]) -> int: + from litellm.litellm_core_utils.token_counter import token_counter + + return token_counter( + model="anthropic/claude-fable-5", + messages=[{"role": "user", "content": content}], + use_default_image_token_count=True, + ) + + +@pytest.mark.parametrize( + "source", + [ + {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}, + {"type": "url", "url": "https://example.com/report.pdf"}, + {"type": "file", "file_id": "file-abc123"}, + ], + ids=["base64", "url", "file"], +) +def test_anthropic_document_block_with_opaque_source_is_priced_like_an_image(source: dict[str, str]): + """ + A `document` whose bytes cannot be tokenized locally must not raise (it 500ed + /v1/messages/count_tokens before) and is priced exactly like an `image` block. + """ + prompt = {"type": "text", "text": "Summarize this file."} + + assert _count_user_content([prompt, {"type": "document", "source": source}]) == _count_user_content( + [prompt, {"type": "image", "source": source}] + ) + + +def test_anthropic_document_block_text_sources_count_their_text(): + """`text` and `content` document sources count the text they carry, as inline text blocks would.""" + prompt = {"type": "text", "text": "Summarize this file."} + body = {"type": "text", "text": "Revenue grew eleven percent while churn fell to two percent."} + picture = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}} + + text_source = {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": body["text"]}} + assert _count_user_content([prompt, text_source]) == _count_user_content([prompt, body]) + + string_content = {"type": "document", "source": {"type": "content", "content": body["text"]}} + assert _count_user_content([prompt, string_content]) == _count_user_content([prompt, body]) + + block_content = {"type": "document", "source": {"type": "content", "content": [body, picture]}} + assert _count_user_content([prompt, block_content]) == _count_user_content([prompt, body, picture]) + + +def test_anthropic_document_title_and_context_add_their_tokens(): + prompt = {"type": "text", "text": "Summarize this file."} + source = {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"} + described = {"type": "document", "source": source, "title": "Q3 board packet", "context": "Shared by finance"} + + assert _count_user_content([prompt, described]) == _count_user_content( + [ + prompt, + {"type": "text", "text": "Q3 board packet"}, + {"type": "text", "text": "Shared by finance"}, + {"type": "document", "source": source}, + ] + ) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py index f39192b171b..6fffead102f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py @@ -184,3 +184,48 @@ def test_transform_request_unsafe_body(client, auth_as, monkeypatch): response = client.post("/utils/transform_request", json=payload) assert response.status_code == 400 assert "unsafe" in response.text or "error" in response.text + + +def test_token_counter_fallback_counts_tools_system_and_anthropic_blocks(client, auth_as, monkeypatch): + """ + Without a provider counter the route falls back to ``litellm.token_counter``. That count + must include the request's tools and system prompt, and Anthropic ``image`` and ``document`` + blocks must be counted instead of turning the whole request into a 500. + """ + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "disable_token_counter", False, raising=False) + system = [{"type": "text", "text": "You are a terse assistant. Answer in one sentence."}] + tools = [ + { + "name": "get_weather", + "description": "Look up the current weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + ] + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this file?"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}}, + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}}, + ], + } + ] + + def count(payload: dict) -> int: + with auth_as(): + response = client.post("/utils/token_counter", json={"model": "claude-fable-5", **payload}) + assert response.status_code == 200, response.text + return response.json()["total_tokens"] + + bare = count({"messages": messages}) + full = count({"messages": messages, "tools": tools, "system": system}) + + assert bare == litellm.token_counter(model="claude-fable-5", messages=messages) + assert full == litellm.token_counter( + model="claude-fable-5", + messages=[{"role": "system", "content": system}, *messages], + tools=tools, + ) + assert full > bare From 83ab87091b3ea06b2b40f76c7797dc4bcb2c55ed Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:10:30 -0700 Subject: [PATCH 13/22] fix(proxy): only attach tools to the count_tokens fallback when counting messages --- litellm/proxy/proxy_server.py | 2 +- .../proxy/proxy_server/test_routes_utils.py | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f107eafd283..68d0960905f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12244,7 +12244,7 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False) typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) ) counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats - list[ChatCompletionToolParam] | None, tools + list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) total_tokens: Final = await asyncify(litellm.token_counter)( model=model_to_use, diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py index 6fffead102f..ea36f31a82f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py @@ -229,3 +229,33 @@ def test_token_counter_fallback_counts_tools_system_and_anthropic_blocks(client, tools=tools, ) assert full > bare + + +def test_token_counter_fallback_prompt_with_tools_does_not_500(client, auth_as, monkeypatch): + """ + Regression: a raw-text ``prompt`` request that also carries ``tools`` (no ``messages``) must + still count. ``litellm.token_counter`` rejects tools on the text path, so the fallback route + only attaches tools when it is counting messages; otherwise this 500'd instead of returning + the plain text count. + """ + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "disable_token_counter", False, raising=False) + prompt = "count the tokens in this sentence please" + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Look up the current weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, + } + ] + + with auth_as(): + response = client.post( + "/utils/token_counter", json={"model": "claude-fable-5", "prompt": prompt, "tools": tools} + ) + + assert response.status_code == 200, response.text + assert response.json()["total_tokens"] == litellm.token_counter(model="claude-fable-5", text=prompt) From 24d226c6c2cba581a3833486ce099d240705fd77 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:28:21 -0700 Subject: [PATCH 14/22] chore(token_counter): drop docstrings and test prose that restated the count_tokens branches --- litellm/litellm_core_utils/token_counter.py | 19 +------- .../litellm_core_utils/test_token_counter.py | 46 +++---------------- .../proxy/proxy_server/test_routes_utils.py | 13 +----- 3 files changed, 10 insertions(+), 68 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 98bcedf43fe..256bee7b348 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -656,11 +656,6 @@ def _validate_anthropic_content(content: Mapping[str, Any]) -> type: def _anthropic_image_source_data( source: AnthropicContentParamSource | AnthropicContentParamSourceUrl | AnthropicContentParamSourceFileId, ) -> str: - """ - Resolve an Anthropic image `source` to the data string `calculate_img_tokens` prices. - - Returns "" for a `file` source, whose bytes the proxy cannot resolve locally. - """ if source["type"] == "base64": data: Final = source.get("data") if not data: @@ -678,12 +673,6 @@ def _count_document_tokens( use_default_image_token_count: bool, default_token_count: int | None, ) -> int: - """ - Count an Anthropic `document` block: its title and context text, plus the source itself. - - Text-bearing sources (`text`, `content`) count their text; opaque ones (`base64`, `url`, - `file`) are priced like an image, since their bytes cannot be tokenized locally. - """ source: Final = document["source"] metadata_tokens: Final = sum( count_function(text) for text in (document.get("title"), document.get("context")) if text @@ -765,13 +754,7 @@ def _count_content_list( use_default_image_token_count: bool, default_token_count: int | None, ) -> int: - """ - Recursively count tokens from a list of content blocks. - - The block union is wider than OpenAI's: the proxy's Anthropic endpoints count - their native blocks through this same helper, so an `image` block is as much - an input here as OpenAI's `image_url`. - """ + """Recursively count tokens from a list of content blocks.""" try: num_tokens = 0 for c in content_list: diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index b8f001240c9..572b505e94c 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1172,16 +1172,7 @@ def test_count_content_list_rejects_unknown_type(): ids=["base64", "url", "file"], ) def test_token_counter_with_anthropic_image_block(source: dict[str, str]): - """ - Anthropic-native `image` blocks must NOT raise, for every source variant. - - Before this fix `_count_content_list` raised - `Invalid content item type: image`. That 500s /v1/messages/count_tokens and - /utils/token_counter, and it makes the router's context-window pre-call - check swallow the error and return every deployment unfiltered, so an - oversized prompt carrying an image is dispatched upstream instead of being - rejected locally. - """ + """Anthropic `image` blocks must count for every source variant, not raise `Invalid content item type` (which the router's context-window pre-call check swallows into an unfiltered dispatch).""" from litellm.constants import DEFAULT_IMAGE_TOKEN_COUNT messages = [ @@ -1205,11 +1196,7 @@ def test_token_counter_with_anthropic_image_block(source: dict[str, str]): def test_anthropic_image_block_matches_equivalent_image_url(): - """ - An Anthropic `image` block must price identically to the OpenAI `image_url` - block carrying the same bytes, so the count does not depend on which - endpoint shape the caller used. - """ + """An Anthropic `image` block prices identically to the OpenAI `image_url` carrying the same bytes.""" anthropic_messages = [ { "role": "user", @@ -1247,11 +1234,7 @@ def test_anthropic_image_block_matches_equivalent_image_url(): def test_anthropic_image_block_nested_in_tool_result(): - """ - An `image` block nested inside a `tool_result.content` list must be counted - too. `_count_anthropic_content` recurses back into `_count_content_list`, so - the nested case failed for the same reason the top-level one did. - """ + """An `image` block nested in a `tool_result.content` list is counted through the same recursion.""" messages = [ { "role": "user", @@ -1292,22 +1275,14 @@ def test_anthropic_image_block_nested_in_tool_result(): ids=["base64", "url", "file"], ) def test_anthropic_image_source_resolves_to_what_the_image_pricer_reads(source: dict[str, str], expected: str): - """ - The image pricer reads either a data URI or a fetchable URL: a base64 source keeps its - media type inside the URI, a url source passes through untouched, and a file source has - no bytes the proxy can measure locally. - """ + """base64 sources become a data URI, url sources pass through, file sources resolve to an empty string.""" from litellm.litellm_core_utils.token_counter import _anthropic_image_source_data assert _anthropic_image_source_data(source) == expected def test_anthropic_image_block_with_empty_base64_data(): - """ - A base64 source carrying no bytes must still price as an image rather than - raise: the block is well-formed enough to count, and an empty `data` only - means there is nothing to measure the dimensions from. - """ + """A base64 source with empty `data` prices as an image rather than raising.""" from litellm.litellm_core_utils.token_counter import _count_content_list tokens = _count_content_list( @@ -1322,11 +1297,7 @@ def test_anthropic_image_block_with_empty_base64_data(): def test_anthropic_image_block_without_source_raises(): - """ - An `image` block with no `source` is malformed, and must fail the same way - the OpenAI `image_url` block with no `url` does - a ValueError the caller - can turn into a 400 - instead of being silently counted as a valid image. - """ + """An `image` block with no `source` raises, matching the OpenAI `image_url`-without-`url` behavior.""" from litellm.litellm_core_utils.token_counter import _count_content_list with pytest.raises(ValueError, match="Error getting number of tokens from content list"): @@ -1369,10 +1340,7 @@ def _count_user_content(content: list[dict]) -> int: ids=["base64", "url", "file"], ) def test_anthropic_document_block_with_opaque_source_is_priced_like_an_image(source: dict[str, str]): - """ - A `document` whose bytes cannot be tokenized locally must not raise (it 500ed - /v1/messages/count_tokens before) and is priced exactly like an `image` block. - """ + """A `document` whose bytes can't be tokenized locally is priced like an `image`, not raised on.""" prompt = {"type": "text", "text": "Summarize this file."} assert _count_user_content([prompt, {"type": "document", "source": source}]) == _count_user_content( diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py index ea36f31a82f..35b5c72f92e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py @@ -187,11 +187,7 @@ def test_transform_request_unsafe_body(client, auth_as, monkeypatch): def test_token_counter_fallback_counts_tools_system_and_anthropic_blocks(client, auth_as, monkeypatch): - """ - Without a provider counter the route falls back to ``litellm.token_counter``. That count - must include the request's tools and system prompt, and Anthropic ``image`` and ``document`` - blocks must be counted instead of turning the whole request into a 500. - """ + """The ``litellm.token_counter`` fallback counts the request's tools and system prompt, and Anthropic ``image``/``document`` blocks, instead of 500ing.""" monkeypatch.setattr(proxy_server, "llm_router", None) monkeypatch.setattr(litellm, "disable_token_counter", False, raising=False) system = [{"type": "text", "text": "You are a terse assistant. Answer in one sentence."}] @@ -232,12 +228,7 @@ def test_token_counter_fallback_counts_tools_system_and_anthropic_blocks(client, def test_token_counter_fallback_prompt_with_tools_does_not_500(client, auth_as, monkeypatch): - """ - Regression: a raw-text ``prompt`` request that also carries ``tools`` (no ``messages``) must - still count. ``litellm.token_counter`` rejects tools on the text path, so the fallback route - only attaches tools when it is counting messages; otherwise this 500'd instead of returning - the plain text count. - """ + """Regression: a ``prompt`` request carrying ``tools`` but no ``messages`` still counts, because the fallback attaches tools only when counting messages (``token_counter`` rejects tools on the text path).""" monkeypatch.setattr(proxy_server, "llm_router", None) monkeypatch.setattr(litellm, "disable_token_counter", False, raising=False) prompt = "count the tokens in this sentence please" From 418b820af43b103f24f5129ec86acb20f7082f83 Mon Sep 17 00:00:00 2001 From: yassin Date: Fri, 28 Aug 2026 14:42:22 +0000 Subject: [PATCH 15/22] fix(proxy): let llm_api virtual keys read /model_group/info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 1 + .../proxy/auth/test_route_checks.py | 31 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1978c0a1b0b..ac98939288e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -566,6 +566,7 @@ class LiteLLMRoutes(enum.Enum): model_info_routes = [ "/model/info", "/v1/model/info", + "/model_group/info", ] llm_api_routes = ( diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 2eab03c2947..71ccef620e5 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -424,6 +424,29 @@ def test_virtual_key_llm_api_routes_rejects_non_get_mcp_server_discovery(route, assert exc_info.value.status_code == 403 +def test_virtual_key_llm_api_routes_allows_model_group_info(): + """Regression test: the UI mints virtual keys with key_type="llm_api", which + maps to allowed_routes=["llm_api_routes"]. The Playground model picker loads + its options from GET /model_group/info, so that key must reach the route or + no model can be selected. The handler already scopes the response to the + models the key can call. + """ + + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["llm_api_routes"], + ) + + assert ( + RouteChecks.is_virtual_key_allowed_to_call_route( + route="/model_group/info", + valid_token=valid_token, + request=_mock_request("GET"), + ) + is True + ) + + @pytest.mark.parametrize( "route", [ @@ -523,7 +546,7 @@ def test_virtual_key_llm_api_routes_allows_model_info(route): assert result is True -@pytest.mark.parametrize("route", ["/model/info", "/v1/model/info"]) +@pytest.mark.parametrize("route", ["/model/info", "/v1/model/info", "/model_group/info"]) def test_model_info_not_classified_as_llm_api(route): """Membership in `llm_api_routes` must not promote /model/info to an `is_llm_api_route()`. That predicate gates DISABLE_LLM_API_ENDPOINTS, @@ -535,10 +558,10 @@ def test_model_info_not_classified_as_llm_api(route): assert RouteChecks.is_llm_api_route(route=route) is False -@pytest.mark.parametrize("route", ["/v2/model/info", "/model_group/info"]) +@pytest.mark.parametrize("route", ["/v2/model/info"]) def test_virtual_key_llm_api_routes_denies_other_model_info_routes(route): - """The grant is scoped to the two /model/info paths. The paginated Admin UI - listing and the model-group endpoint stay outside it. + """The grant covers the model metadata reads an AI API key needs. The + paginated Admin UI listing stays outside it. """ valid_token = UserAPIKeyAuth( From 021e03fe904cfcadc6af8ae0a4e4047adc73c16f Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 09:03:57 -0700 Subject: [PATCH 16/22] test(passthrough): serve the passthrough target locally instead of calling OpenAI Greptile flagged this test as coupled to OpenAI's availability. The coupling was not the status assertion it pointed at, and it predates this PR: the test it replaced called /v1/assistants live the same way, and pass_through_endpoints gates success logging on `response.status_code < 400`, so an upstream outage has always meant no log fires and the payload assertions fail regardless. The target is now a local HTTP server on an ephemeral port, so the test is offline either way. It still exercises the generic passthrough handler, since _is_supported_openai_endpoint does not claim a 127.0.0.1 URL any more than it claimed /v1/moderations, and it now also asserts what the upstream actually received rather than only what came back. respx was the obvious approach and does not work here: it patches httpx transports, and the passthrough issues its request through the custom aiohttp transport, so the call went to the real api.openai.com and returned 401 while respx sat unused. Mutation checked: gating off the success enqueue fails the test, and tampering with the logged response body fails it. --- .../test_custom_logger_passthrough.py | 63 ++++++++++++++++--- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py index d7add68753b..a7d6378fa18 100644 --- a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py +++ b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py @@ -1,5 +1,6 @@ import json -import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Optional from fastapi import Request import pytest @@ -29,14 +30,59 @@ class TestCustomLogger(CustomLogger): self.logged_kwargs = kwargs +UPSTREAM_RESPONSE_BODY = { + "id": "modr-abc123", + "model": "omni-moderation-latest", + "results": [ + { + "flagged": False, + "categories": {"violence": False}, + "category_scores": {"violence": 1.2e-06}, + } + ], +} + + +@pytest.fixture +def upstream(): + received: dict = {} + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self): + body = self.rfile.read(int(self.headers.get("content-length", 0) or 0)) + received["path"] = self.path + received["body"] = json.loads(body or b"{}") + payload = json.dumps(UPSTREAM_RESPONSE_BODY).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"http://127.0.0.1:{server.server_port}", received + finally: + server.shutdown() + server.server_close() + + @pytest.mark.asyncio -async def test_untracked_openai_route_passthrough_logging(): +async def test_untracked_openai_route_passthrough_logging(upstream): """Keep this on a route `_is_supported_openai_endpoint` does not claim, or the OpenAI-specific handler takes over and the generic payload stops being exercised.""" + base_url, upstream_received = upstream + test_custom_logger = TestCustomLogger() litellm._async_success_callback = [test_custom_logger] - TARGET_URL = "https://api.openai.com/v1/moderations" + TARGET_URL = f"{base_url}/v1/moderations" REQUEST_BODY = { "model": "omni-moderation-latest", "input": "I want to bake a cake for my friend's birthday.", @@ -52,17 +98,14 @@ async def test_untracked_openai_route_passthrough_logging(): "query_string": b"", "headers": [ (b"content-type", b"application/json"), - ( - b"authorization", - f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(), - ), + (b"authorization", b"Bearer sk-test-passthrough"), ], }, ), target=TARGET_URL, custom_headers={ "Content-Type": "application/json", - "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}", + "Authorization": "Bearer sk-test-passthrough", }, user_api_key_dict=UserAPIKeyAuth( api_key="test", @@ -79,6 +122,8 @@ async def test_untracked_openai_route_passthrough_logging(): print("result status code", result.status_code) print("result content", result.body) + assert upstream_received.get("path") == "/v1/moderations" + assert upstream_received.get("body") == REQUEST_BODY assert result.status_code == 200 await asyncio.sleep(1) @@ -93,5 +138,5 @@ async def test_untracked_openai_route_passthrough_logging(): assert passthrough_logging_payload["request_method"] == TARGET_METHOD client_facing_response_body = json.loads(result.body) - assert client_facing_response_body["results"] + assert client_facing_response_body == UPSTREAM_RESPONSE_BODY assert passthrough_logging_payload["response_body"] == client_facing_response_body From cddf1f6c03990a2b205b1560ae065dec62326671 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 09:05:56 -0700 Subject: [PATCH 17/22] chore(lint): ratchet the TQ ceilings for the passthrough isolation --- test-quality-budget.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index cab7a027e93..db96156e4d9 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -1,6 +1,6 @@ { "TQ001": { - "limit": 739 + "limit": 736 }, "TQ002": { "limit": 742 @@ -12,7 +12,7 @@ "limit": 469 }, "TQ005": { - "limit": 2401 + "limit": 2399 }, "TQ006": { "limit": 34 From 13a0976bb6b0e0f29f1b3ccb164f929fee925a81 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 09:11:48 -0700 Subject: [PATCH 18/22] test(passthrough): name the test for what it now covers The target is a local server, so the OpenAI host check already excludes it and the docstring's claim about the route path no longer holds. --- .../test_custom_logger_passthrough.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py index a7d6378fa18..70fc8f9ccf2 100644 --- a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py +++ b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py @@ -74,9 +74,9 @@ def upstream(): @pytest.mark.asyncio -async def test_untracked_openai_route_passthrough_logging(upstream): - """Keep this on a route `_is_supported_openai_endpoint` does not claim, or the - OpenAI-specific handler takes over and the generic payload stops being exercised.""" +async def test_passthrough_logging_payload_for_a_route_no_provider_handler_claims( + upstream, +): base_url, upstream_received = upstream test_custom_logger = TestCustomLogger() From 2b0932f930bc0e884aaf5069adce02c04940d470 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 09:35:12 -0700 Subject: [PATCH 19/22] test(access-groups): give each xdist worker its own fixture ids Every test in the file seeds, reads and deletes the same fixed group and team ids, and auth_ui_unit_tests runs pytest with -n 2. Two tests landing on the two workers at once tread on each other: one worker's _clean_db DELETE wipes rows the other just seeded, and its sync writes land in the other's read. Both shapes showed up on 13a0976bb6, a commit that renames a passthrough test and nothing else. test_reconcile_is_idempotent... read back an empty table, and test_reconcile_handles_a_null_array_column read the idempotent test's team on its own second group. Scoping the ids to PYTEST_XDIST_WORKER keeps each worker in its own rows. Tests on one worker still run in sequence, so no isolation is lost. Reproduced against a local Postgres: -n 2 failed 6 out of 6 runs before, passed 6 out of 6 after, and serial runs are green either way. Stripping the COALESCE guard from the mirror's SQL still fails the suite, so the ids are all that changed. --- tests/proxy_admin_ui_tests/test_access_group_team_sync.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py index b72a1453576..ceb0dbf6749 100644 --- a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py +++ b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py @@ -23,9 +23,10 @@ from litellm.proxy.management_helpers.access_group_team_sync import ( sync_team_access_group_membership, ) -TEAM = "ags-team-a" -OTHER_TEAM = "ags-team-b" -GROUPS = ("ags-group-1", "ags-group-2", "ags-group-3") +_XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "master") +TEAM = f"ags-team-a-{_XDIST_WORKER}" +OTHER_TEAM = f"ags-team-b-{_XDIST_WORKER}" +GROUPS = tuple(f"ags-group-{n}-{_XDIST_WORKER}" for n in (1, 2, 3)) _DELETE_SEEDED = 'DELETE FROM "LiteLLM_AccessGroupTable" WHERE access_group_id = ANY($1::TEXT[])' _DELETE_TEAMS = 'DELETE FROM "LiteLLM_TeamTable" WHERE team_id = ANY($1::TEXT[])' From 721db0f03ecfe069847b21ad2d383617176bd422 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 28 Aug 2026 09:45:11 -0700 Subject: [PATCH 20/22] feat(ui): edit the auto-router tier set with custom classifier-defined tiers (#38603) * feat(ui): edit the auto-router tier set with custom classifier-defined tiers The editor over the model layer beneath it. An Edit tiers button turns the tier list into an editor: a tier takes a name, a classifier definition, and models, between two and eight rows. Restore defaults resets to the built-in four rather than stacking them on top. Keyword rules follow a rename, an orphaned rule blocks the save, and both forms dry-run the exact payload against the backend validator before writing. The edit modal hydrates a stored custom set into rows, and an untouched open-and-save round-trips byte-identically, per-model reasoning efforts included. A form that never opens the editor submits the same bytes as before. The cost-optimization tier chart renders arbitrary tier names: the guard that returned no models for a non-built-in name is gone, and the fixed four-color array gives way to the shared cycle. * refactor(ui): extract tier editor sections to clear new lint warnings * test(ui): drop narration comments per repo convention * fix(ui): default editingTiers so the build's type check passes * fix(ui): restore the mid-dry-run submit guard and its regression tests --- .../CostOptimizationView.activity.test.tsx | 2 + .../_components/TierTurnsChart.test.tsx | 14 + .../_components/TierTurnsChart.tsx | 7 +- .../add_model/ClassificationMethodConfig.tsx | 131 +++--- .../add_model/ComplexityRouterConfig.test.tsx | 213 +++++++++ .../add_model/ComplexityRouterConfig.tsx | 435 ++++++++++++++---- .../components/add_model/KeywordTierRules.tsx | 9 +- .../components/add_model/TierRestrictions.tsx | 24 + .../add_model/add_auto_router_tab.test.tsx | 87 ++-- .../add_model/add_auto_router_tab.tsx | 49 +- ...d_updated_complexity_router_config.test.ts | 52 ++- .../edit_auto_router_modal.test.tsx | 89 +++- .../edit_auto_router_modal.tsx | 75 ++- 13 files changed, 935 insertions(+), 252 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/TierRestrictions.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx index 2d46ca48adb..1cc7bec13d1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx @@ -32,6 +32,8 @@ vi.mock("@/components/shared/charts", () => ({ DonutChart: () =>
, BarChart: () =>
, CustomLegend: () =>
, + chartColorValue: (color: string) => color, + DEFAULT_COLOR_CYCLE: ["blue", "cyan", "sky", "indigo", "violet", "purple", "fuchsia", "slate"], SEQUENTIAL_COLOR_RAMP: ["indigo"], })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx index 057eb54ee4e..da4af8baf29 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx @@ -6,6 +6,7 @@ import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useMod vi.mock("@/components/shared/charts", () => ({ DonutChart: ({ label }: { label: string }) =>
{label}
, + DEFAULT_COLOR_CYCLE: ["blue", "cyan", "sky", "indigo", "violet", "purple", "fuchsia", "slate"], SEQUENTIAL_COLOR_RAMP: ["indigo", "blue"], chartColorValue: (color: string) => color, })); @@ -111,6 +112,19 @@ describe("TierTurnsChart", () => { expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument(); }); + it("lists a custom tier's models, which the built-in name guard used to hide", () => { + render( + , + ); + + expect(screen.getByText(/SECURITY_REVIEW/)).toBeInTheDocument(); + expect(screen.getByText("o1-preview")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument(); + }); + it("omits the model line for a tier with no configured models", () => { render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx index e55ebc07656..44cca6331b8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx @@ -11,7 +11,7 @@ import { type ComplexityTiers, } from "@/components/add_model/ComplexityRouterConfig"; import { normalizeTierModels } from "@/components/add_model/complexity_router_tiers"; -import { chartColorValue, DonutChart, type ChartColor } from "@/components/shared/charts"; +import { chartColorValue, DEFAULT_COLOR_CYCLE, DonutChart } from "@/components/shared/charts"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { viewGroup, type BenchmarkView } from "./autoRouterBenchmarks"; @@ -71,7 +71,6 @@ const tierModelsFor = ( routerType: string, autoRouters: readonly AutoRouterDeployment[], ): string[] => { - if (!isComplexityTier(tier)) return []; const deployment = deploymentFor(routerName, routerType, autoRouters); if (!deployment) return []; const config = asRecord(deployment.litellm_params?.complexity_router_config); @@ -84,8 +83,6 @@ interface TierTurnsChartProps { autoRouters: readonly AutoRouterDeployment[]; } -const TIER_DONUT_COLORS: readonly ChartColor[] = ["#c7d2fe", "#1e293b", "#d4b483", "#87a878"]; - const TierTurnsChart: React.FC = ({ view, autoRouters }) => { const group = viewGroup(view); const entries = Object.entries(group?.tier_turns ?? {}).filter(([, turns]) => turns > 0); @@ -98,7 +95,7 @@ const TierTurnsChart: React.FC = ({ view, autoRouters }) => turns, models: tierModelsFor(tier, group.router_name, group.router_type, autoRouters), })); - const colors = slices.map((_, idx) => TIER_DONUT_COLORS[idx % TIER_DONUT_COLORS.length]); + const colors = slices.map((_, idx) => DEFAULT_COLOR_CYCLE[idx % DEFAULT_COLOR_CYCLE.length]); return ( diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index cea967f5966..2a25995a442 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -10,6 +10,7 @@ import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; import { Switch } from "@/components/ui/switch"; import React from "react"; import ClassifierPromptEditor from "./ClassifierPromptEditor"; +import { Restricted, RestrictedSection, restrictedBy } from "./TierRestrictions"; import HeuristicScoringConfig from "./HeuristicScoringConfig"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { @@ -31,6 +32,7 @@ import { usesLlmClassifier, DEFAULT_HEURISTIC_FIRST_MAX_TIER, HEURISTIC_FIRST_MAX_TIER_KEYS, + effectiveClassifierType, } from "./ComplexityRouterConfig"; const DEFAULT_SCORING_EXPLANATION = @@ -89,18 +91,21 @@ const boundaryRanges = ( const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = ({ value }) => { // The shipped boundaries come from the proxy, so this card cannot state ranges the router stopped using. const { data: scorerDefaults, isError } = useComplexityScorerDefaults(); + const scorerRuns = heuristicScoringRole(value) !== "never"; const ranges = boundaryRanges( scorerDefaults?.tier_boundaries, value.tier_boundaries, value.reasoning_override_min_score, ); + if (value.custom_tier_set) return null; + return ( How Classification Works {scoringExplanation(value)} - {ranges && ( + {scorerRuns && ranges && (
  • {effectiveTierLabel("SIMPLE", value.tier_labels)}: Score < {ranges.simpleMedium} @@ -141,6 +146,54 @@ interface ClassificationMethodConfigProps { defaultModel?: string; } +const ClassifierTypeRadios: React.FC<{ + value: ComplexityRouterConfigValue; + classifierType: ClassifierType; + onTypeChange: (classifierType: ClassifierType) => void; +}> = ({ value, classifierType, onTypeChange }) => { + const scorerLocked = Boolean(value.custom_tier_set); + const scorerLockedReason = restrictedBy(value, "heuristicClassifier")?.reason; + return ( + onTypeChange(classifierType as ClassifierType)} + className="w-full" + > +
    + + + + + + + +
    +     + ); +}; + const ClassificationMethodConfig: React.FC = ({ value, onChange, @@ -151,8 +204,9 @@ const ClassificationMethodConfig: React.FC = ({ defaultModel, }) => { const hasDefaultModel = Boolean(defaultModel); + const classifierType = effectiveClassifierType(value); const classifierModelMissing = - showValidationErrors && usesLlmClassifier(value.classifier_type) && !value.classifier_llm_config?.model; + showValidationErrors && usesLlmClassifier(classifierType) && !value.classifier_llm_config?.model; const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS; const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS; @@ -264,41 +318,9 @@ const ClassificationMethodConfig: React.FC = ({ return ( <> - handleClassifierTypeChange(classifierType as ClassifierType)} - className="w-full" - > -
    - - - -
    -     + - {value.classifier_type === "heuristic_first" && ( + {classifierType === "heuristic_first" && (
    Decide locally up to = ({ onValueChange={(preset: ClassificationRubric | null) => preset && handleClassificationRubricChange(preset) } - disabled={usesCustomPrompt} + disabled={usesCustomPrompt || Boolean(value.custom_tier_set)} > @@ -388,23 +413,25 @@ const ClassificationMethodConfig: React.FC = ({ - {usesCustomPrompt - ? "Not in use: the custom prompt below is the classifier's entire rubric." - : CLASSIFICATION_RUBRIC_DESCRIPTIONS[classificationRubric].description} + {restrictedBy(value, "classificationRubric")?.reason ?? + (usesCustomPrompt + ? "Not in use: the custom prompt below is the classifier's entire rubric." + : CLASSIFICATION_RUBRIC_DESCRIPTIONS[classificationRubric].description)}
    Classifier Prompt - + + +
    -
    - If the classifier fails + handleClassifierFallbackChange(fallback as ClassifierFallback)} @@ -439,7 +466,7 @@ const ClassificationMethodConfig: React.FC = ({ Applies when the classifier call errors, times out, or returns an unparseable response. -
    +
    Context Window Size { expect(screen.queryByText("Custom Technical Keywords")).not.toBeInTheDocument(); }); }); + +describe("ComplexityRouterConfig tier editing", () => { + const renderEditor = ( + value?: ComplexityRouterConfigValue, + props: Partial> = {}, + ) => { + const onChange = vi.fn(); + const view = renderWithProviders( + , + ); + return { ...view, committed: () => onChange.mock.calls[0][0] as ComplexityRouterConfigValue, onChange }; + }; + + const customValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-4", timeout_ms: 3000 }, + custom_tier_set: { + tiers: [ + { id: "CASUAL", name: "CASUAL", definition: "small talk", models: ["gpt-3.5-turbo"] }, + { id: "sec", name: "SECURITY_REVIEW", definition: "audits", models: ["gpt-4"] }, + ], + fallback_tier_id: "CASUAL", + }, + }; + + it("offers Edit tiers only when the parent owns the editor flag", () => { + renderWithProviders(); + expect(screen.queryByRole("button", { name: "Edit tiers" })).not.toBeInTheDocument(); + }); + + it("renders the four built-in tiers before any edit, unchanged", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: "Edit tiers" })).toBeInTheDocument(); + expect(screen.getByText("Tier 1 of 4", { exact: false })).toHaveTextContent("SIMPLE"); + }); + + it("adds a row and moves the form into an edited tier set, which the built-in record never leaves", () => { + const { committed } = renderEditor(); + fireEvent.click(screen.getByRole("button", { name: "Add tier" })); + const next = committed(); + expect(next.custom_tier_set?.tiers).toHaveLength(5); + expect(next.tiers).toEqual(defaultValue.tiers); + }); + + it("renames a built-in tier straight from the editor, which is what makes the set custom", () => { + const { committed } = renderEditor(); + fireEvent.change(screen.getByLabelText("Name for tier 3"), { target: { value: "SECURITY_REVIEW" } }); + const next = committed(); + expect(next.custom_tier_set?.tiers.map((row) => row.name)).toEqual([ + "SIMPLE", + "MEDIUM", + "SECURITY_REVIEW", + "REASONING", + ]); + expect(next.tiers).toEqual(defaultValue.tiers); + }); + + it("opening the editor and changing nothing leaves the router on the built-in tiers", () => { + const { onChange } = renderEditor(); + expect(screen.getByRole("button", { name: "Done" })).toBeEnabled(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("swaps the display-name field for the tier-name field while the editor is open", () => { + const { rerender } = renderWithProviders(); + expect(screen.getByLabelText("Display name for the Simple tier")).toBeInTheDocument(); + rerender(); + expect(screen.queryByLabelText("Display name for the Simple tier")).not.toBeInTheDocument(); + expect(screen.getByLabelText("Name for tier 1")).toBeInTheDocument(); + }); + + it("replaces the prompt editor with the reason an edited tier set forbids it", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByText("A replacement prompt drops the tier bullets", { exact: false })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Change default prompt" })).not.toBeInTheDocument(); + }); + + it("drops the scorer card entirely once an edited tier set replaces the heuristic", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument(); + expect(screen.queryByText("scores each request across 7 dimensions", { exact: false })).not.toBeInTheDocument(); + }); + + it("keeps the scorer card on a built-in router, whose tiers the score still decides", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByText("How Classification Works")).toBeInTheDocument(); + expect(screen.getByText("scores each request across 7 dimensions", { exact: false })).toBeInTheDocument(); + }); + + it("says why a custom row is blocked instead of only reddening its border", () => { + const missingDefinition: ComplexityRouterConfigValue = { + ...customValue, + custom_tier_set: { + tiers: [customValue.custom_tier_set!.tiers[0], { id: "b", name: "AUDIT", definition: "", models: ["gpt-4"] }], + fallback_tier_id: "CASUAL", + }, + }; + renderEditor(missingDefinition, { showValidationErrors: true }); + expect(screen.getByText("A definition is required", { exact: false })).toBeInTheDocument(); + }); + + it("keeps Done disabled while a row is incomplete and says what is missing", async () => { + const incomplete: ComplexityRouterConfigValue = { + ...customValue, + custom_tier_set: { + tiers: [customValue.custom_tier_set!.tiers[0], { id: "new", name: "", definition: "", models: [] }], + fallback_tier_id: "CASUAL", + }, + }; + renderEditor(incomplete); + expect(screen.getByRole("button", { name: "Done" })).toBeDisabled(); + }); + + it("enables Done once every row carries a name, a definition and a model", () => { + renderEditor(customValue); + expect(screen.getByRole("button", { name: "Done" })).toBeEnabled(); + }); + + it("refuses to remove a row that would take the set below the backend's minimum", () => { + renderEditor(customValue); + expect(screen.getByRole("button", { name: "Remove the CASUAL tier" })).toBeDisabled(); + }); + + it("keeps a definition on one line, because the backend rejects a newline in it", () => { + const { committed } = renderEditor(customValue); + fireEvent.change(screen.getByLabelText("Definition for tier 2"), { target: { value: "audits\nand reviews" } }); + const next = committed(); + expect(next.custom_tier_set?.tiers[1].definition).toBe("audits and reviews"); + }); + + it("moves a keyword rule with the tier it points at when that tier is renamed", () => { + const onKeywordTierRulesChange = vi.fn(); + renderWithProviders( + , + ); + fireEvent.change(screen.getByLabelText("Name for tier 2"), { target: { value: "AUDIT" } }); + expect(onKeywordTierRulesChange).toHaveBeenCalledWith([{ id: "r1", keywords: ["audit"], tier: "AUDIT" }]); + }); + + it("re-points the fallback tier when the row it named is removed, never leaving it dangling", () => { + const threeRows: ComplexityRouterConfigValue = { + ...customValue, + custom_tier_set: { + tiers: [ + ...customValue.custom_tier_set!.tiers, + { id: "third", name: "MEDIUM", definition: "", models: ["gpt-4"] }, + ], + fallback_tier_id: "sec", + }, + }; + const { committed } = renderEditor(threeRows); + fireEvent.click(screen.getByRole("button", { name: "Remove the SECURITY_REVIEW tier" })); + const next = committed(); + expect(next.custom_tier_set?.tiers.some((row) => row.id === next.custom_tier_set?.fallback_tier_id)).toBe(true); + }); + + it("turns off a plan-mode floor whose row was removed, rather than leaving it pointing at nothing", () => { + const withFloor: ComplexityRouterConfigValue = { + ...customValue, + plan_mode_min_tier: "sec", + custom_tier_set: { + tiers: [ + ...customValue.custom_tier_set!.tiers, + { id: "third", name: "BULK", definition: "d", models: ["gpt-4"] }, + ], + fallback_tier_id: "CASUAL", + }, + }; + const { committed } = renderEditor(withFloor); + fireEvent.click(screen.getByRole("button", { name: "Remove the SECURITY_REVIEW tier" })); + expect(committed().plan_mode_min_tier).toBeUndefined(); + }); + + it("replaces the display-name inputs with the reason an edited tier set forbids them", () => { + renderWithProviders(); + expect(screen.queryByLabelText("Display name for the Simple tier")).not.toBeInTheDocument(); + expect(screen.getByText("Display names rename the built-in tiers", { exact: false })).toBeInTheDocument(); + expect(screen.getByLabelText("Fallback tier")).toBeInTheDocument(); + }); + + it("disables session pinning and says why, rather than letting a stripped value look saved", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Affinity")); + expect(screen.getByLabelText("Pin a session to its first model")).toHaveAttribute("data-disabled"); + expect( + screen.getByText("Session pinning escalates along the built-in tier ladder", { exact: false }), + ).toBeInTheDocument(); + }); + + it("leaves built-in routers with their display-name inputs and no restriction copy", () => { + renderWithProviders(); + expect(screen.getByLabelText("Display name for the Simple tier")).toBeInTheDocument(); + expect(screen.queryByText("Display names rename the built-in tiers", { exact: false })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 9894e26d552..e0371806ea7 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -2,32 +2,50 @@ import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; import { SearchSelect } from "@/components/shared/SearchSelect"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { ChevronRight, Info, X } from "lucide-react"; +import { ChevronRight, Info, Plus, Trash2, X } from "lucide-react"; import { Switch } from "@/components/ui/switch"; import { Card, CardContent } from "@/components/ui/card"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { Separator } from "@/components/ui/separator"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { + type CustomTierSet, + type TierRow, + MAX_TIER_COUNT, + MAX_TIER_DEFINITION_CHARS, + MAX_TIER_NAME_CHARS, + MIN_TIER_COUNT, + TIER_ORDER, + activeTierName, + activeTierRows, + getCustomTierRowsError, + isBuiltInTierName, + resolveComplexityDefaultModel, +} from "./tier_rows"; import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import { Restricted, restrictedBy } from "./TierRestrictions"; +import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { REASONING_EFFORT_OPTIONS, ReasoningEffort, TierModelParamsByTier, - pruneTierModelParams, setTierModelReasoningEffort, + tierRowLabel, } from "./complexity_router_tiers"; import TierModelEffortRows from "./TierModelEffortRows"; import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; -import { type CustomTierSet, type TierRow, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows"; -export type { CustomTierSet, TierRow } from "./tier_rows"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; +export type { CustomTierSet, TierRow } from "./tier_rows"; export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000; export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5; @@ -141,12 +159,210 @@ export const effectiveClassifierType = ( value: Pick, ): ClassifierType => (value.custom_tier_set ? "llm" : value.classifier_type); +const rowOrigin = (row: TierRow, editing: boolean): string => { + if (!editing) return row.id; + return isBuiltInTierName(row.name) ? "built-in" : "custom"; +}; + +const defaultModelPlaceholderFor = (derivedDefaultModel: string | undefined, isCustomSet: boolean): string => { + if (derivedDefaultModel) return `Derived from tiers: ${derivedDefaultModel}`; + return isCustomSet ? "Add a model to your fallback tier" : "Add a model to the Simple or Medium tier"; +}; + +const builtInTierInfo = (rowId: string): { label: string; description: string; examples: string } | undefined => { + const builtIn = TIER_ORDER.find((tier) => tier === rowId); + return builtIn ? TIER_DESCRIPTIONS[builtIn] : undefined; +}; + +const TierConfigIntro: React.FC<{ value: ComplexityRouterConfigValue }> = ({ value }) => ( + <> + + {heuristicScoringRole(value) === "never" + ? "The complexity router classifies each request with your classifier model and routes it to that tier. Configure which model(s) handle each tier." + : "The complexity router automatically classifies requests by complexity using rule-based scoring (no API calls, <1ms latency). Configure which model(s) handle each tier."} + + + + {restrictedBy(value, "displayNames")?.reason ?? + "Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how requests are classified, and callers never see these names."} + {!value.custom_tier_set && + usesLlmClassifier(value.classifier_type) && + " Your classifier model reads these names, so clearer ones can sharpen its choices."} + + +); + +const TierSetToolbar: React.FC<{ + editing: boolean; + isCustomSet: boolean; + rowCount: number; + rowsError: string | null; + onEditingChange: ((editing: boolean) => void) | undefined; + onAdd: () => void; + onRestore: () => void; +}> = ({ editing, isCustomSet, rowCount, rowsError, onEditingChange, onAdd, onRestore }) => ( + <> +
    + {editing ? ( + <> + + + + + {isCustomSet && ( + + )} + + ) : ( + onEditingChange && ( + + ) + )} +
    + {editing && ( + + Add or remove tiers to define your own set. Every custom tier needs a definition the LLM classifier routes on, + and an edited set requires the LLM classification method + + )} + +); + +const FallbackTierField: React.FC<{ + rows: readonly TierRow[]; + fallbackTierId: string; + onValueChange: (rowId: string) => void; +}> = ({ rows, fallbackTierId, onValueChange }) => ( +
    +
    + Fallback Tier + + + +
    + activeTierName(row)).map((row) => ({ value: row.id, label: activeTierName(row) }))} + value={fallbackTierId || null} + onValueChange={onValueChange} + placeholder="Pick the tier classifier failures route to" + /> +
    +); + +const TierRowHeader: React.FC<{ + row: TierRow; + index: number; + rowCount: number; + label: string; + description: string | undefined; + editing: boolean; + isCustomSet: boolean; + onRemove: () => void; +}> = ({ row, index, rowCount, label, description, editing, isCustomSet, onRemove }) => ( +
    + {label} Tier + + + + + Tier {index + 1} of {rowCount} · {rowOrigin(row, isCustomSet)} + + {editing && ( + + )} +
    +); + +const TierRowEditFields: React.FC<{ + row: TierRow; + index: number; + definitionMissing: boolean; + onPatch: (patch: Partial>) => void; +}> = ({ row, index, definitionMissing, onPatch }) => ( + <> + onPatch({ name: event.target.value })} + placeholder="Tier name, e.g. SECURITY_REVIEW" + aria-label={`Name for tier ${index + 1}`} + maxLength={MAX_TIER_NAME_CHARS} + className="mb-2" + /> +