scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

fix(proxy): include provider prefix in kwargs['model'] for Anthropic passthrough metrics

Browse source 
The Anthropic passthrough logging handler set kwargs["model"] to the
raw model ID from the API response (e.g. "global.anthropic.claude-opus-4-6-v1")
instead of the provider-prefixed form ("bedrock/global.anthropic.claude-opus-4-6-v1").

This caused Prometheus to emit duplicate metric series for the same model —
one from /chat/completions (with prefix) and one from /v1/messages (without).

Use the already-computed model_for_cost which carries the correct prefix.

Fixes #25250
This commit is contained in:
Kiyeon Jeon 2026-04-10 23:42:51 +09:00
parent 9e6d2d2069
commit c7002ea2a6
2 changed files with 189 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -143,7 +143,7 @@ class AnthropicPassthroughLoggingHandler:
)
kwargs["response_cost"] = response_cost
kwargs["model"] = model
kwargs["model"] = model_for_cost
passthrough_logging_payload: Optional[PassthroughStandardLoggingPayload] = ( # type: ignore
kwargs.get("passthrough_logging_payload")
)
@ -168,9 +168,9 @@ class AnthropicPassthroughLoggingHandler:
litellm_model_response.model = model
logging_obj.model_call_details["model"] = model
if not logging_obj.model_call_details.get("custom_llm_provider"):
logging_obj.model_call_details[
"custom_llm_provider"
] = litellm.LlmProviders.ANTHROPIC.value
logging_obj.model_call_details["custom_llm_provider"] = (
litellm.LlmProviders.ANTHROPIC.value
)
return kwargs
except Exception as e:
verbose_proxy_logger.exception(

250
tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py
View file

@ -184,9 +184,13 @@ class TestAnthropicLoggingHandlerModelFallback:
logging_obj = self._create_mock_logging_obj() # Empty dict
model = request_body.get("model", "")
if not model and hasattr(logging_obj, 'model_call_details') and logging_obj.model_call_details.get('model'):
model = logging_obj.model_call_details.get('model')
if (
not model
and hasattr(logging_obj, "model_call_details")
and logging_obj.model_call_details.get("model")
):
model = logging_obj.model_call_details.get("model")
assert model == "" # Should remain empty
@ -318,6 +322,108 @@ class TestAzureAnthropicCostCalculation:
assert call_kwargs["custom_llm_provider"] == "azure_ai"
class TestAnthropicPassthroughModelPrefixInKwargs:
"""Regression tests for #25250 — kwargs['model'] must carry provider prefix."""
def _create_mock_logging_obj(
self, model: str = None, custom_llm_provider: str = None
) -> MagicMock:
mock = MagicMock()
details: Dict[str, Any] = {}
if model:
details["model"] = model
if custom_llm_provider:
details["custom_llm_provider"] = custom_llm_provider
mock.model_call_details = details
mock.litellm_call_id = "test-call-id"
return mock
@patch("litellm.completion_cost")
def test_kwargs_model_includes_provider_prefix(self, mock_completion_cost):
"""kwargs['model'] should contain the provider-prefixed name so
Prometheus and other callbacks emit consistent metric labels."""
from litellm.types.utils import ModelResponse
mock_completion_cost.return_value = 0.001
logging_obj = self._create_mock_logging_obj(
model="global.anthropic.claude-opus-4-6-v1",
custom_llm_provider="bedrock",
)
mock_response = MagicMock(spec=ModelResponse)
mock_response.id = "test-id"
mock_response.model = "global.anthropic.claude-opus-4-6-v1"
kwargs: Dict[str, Any] = {}
AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload(
litellm_model_response=mock_response,
model="global.anthropic.claude-opus-4-6-v1",
kwargs=kwargs,
start_time=datetime.now(),
end_time=datetime.now(),
logging_obj=logging_obj,
)
assert kwargs["model"] == "bedrock/global.anthropic.claude-opus-4-6-v1"
@patch("litellm.completion_cost")
def test_kwargs_model_no_duplicate_prefix(self, mock_completion_cost):
"""Provider prefix should not be duplicated if already present."""
from litellm.types.utils import ModelResponse
mock_completion_cost.return_value = 0.001
logging_obj = self._create_mock_logging_obj(
model="bedrock/global.anthropic.claude-opus-4-6-v1",
custom_llm_provider="bedrock",
)
mock_response = MagicMock(spec=ModelResponse)
mock_response.id = "test-id"
kwargs: Dict[str, Any] = {}
AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload(
litellm_model_response=mock_response,
model="bedrock/global.anthropic.claude-opus-4-6-v1",
kwargs=kwargs,
start_time=datetime.now(),
end_time=datetime.now(),
logging_obj=logging_obj,
)
assert kwargs["model"] == "bedrock/global.anthropic.claude-opus-4-6-v1"
@patch("litellm.completion_cost")
def test_kwargs_model_no_provider(self, mock_completion_cost):
"""Without custom_llm_provider, kwargs['model'] should stay as-is."""
from litellm.types.utils import ModelResponse
mock_completion_cost.return_value = 0.001
logging_obj = self._create_mock_logging_obj(
model="claude-3-sonnet-20240229",
)
mock_response = MagicMock(spec=ModelResponse)
mock_response.id = "test-id"
kwargs: Dict[str, Any] = {}
AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload(
litellm_model_response=mock_response,
model="claude-3-sonnet-20240229",
kwargs=kwargs,
start_time=datetime.now(),
end_time=datetime.now(),
logging_obj=logging_obj,
)
assert kwargs["model"] == "claude-3-sonnet-20240229"
class TestAnthropicBatchPassthroughCostTracking:
"""Test cases for Anthropic batch passthrough cost tracking functionality"""
@ -339,10 +445,10 @@ class TestAnthropicBatchPassthroughCostTracking:
"errored": 0,
"expired": 0,
"processing": 1,
"succeeded": 0
"succeeded": 0,
},
"results_url": "https://api.anthropic.com/v1/messages/batches/msgbatch_01Wj7gkQk7gn4MpAKR8ZEDU2/results",
"type": "message_batch"
"type": "message_batch",
}
return mock_response
@ -364,21 +470,20 @@ class TestAnthropicBatchPassthroughCostTracking:
"custom_id": "my-custom-id-1",
"params": {
"max_tokens": 1024,
"messages": [
{
"content": "Hello, world",
"role": "user"
}
],
"model": "claude-sonnet-4-5-20250929"
}
"messages": [{"content": "Hello, world", "role": "user"}],
"model": "claude-sonnet-4-5-20250929",
},
}
]
}
@patch('litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler._store_batch_managed_object')
@patch('litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler.get_actual_model_id_from_router')
@patch('litellm.llms.anthropic.batches.transformation.AnthropicBatchesConfig')
@patch(
"litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler._store_batch_managed_object"
)
@patch(
"litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler.get_actual_model_id_from_router"
)
@patch("litellm.llms.anthropic.batches.transformation.AnthropicBatchesConfig")
def test_batch_creation_handler_success(
self,
mock_batches_config,
@ -386,14 +491,14 @@ class TestAnthropicBatchPassthroughCostTracking:
mock_store_batch,
mock_httpx_response,
mock_logging_obj,
mock_request_body
mock_request_body,
):
"""Test successful batch creation and managed object storage"""
from litellm.types.utils import LiteLLMBatch
# Setup mocks
mock_get_model_id.return_value = "claude-sonnet-4-5-20250929"
mock_batch_response = LiteLLMBatch(
id="msgbatch_01Wj7gkQk7gn4MpAKR8ZEDU2",
object="batch",
@ -416,11 +521,13 @@ class TestAnthropicBatchPassthroughCostTracking:
request_counts={"total": 1, "completed": 0, "failed": 0},
metadata={},
)
mock_batches_config_instance = MagicMock()
mock_batches_config_instance.transform_retrieve_batch_response.return_value = mock_batch_response
mock_batches_config_instance.transform_retrieve_batch_response.return_value = (
mock_batch_response
)
mock_batches_config.return_value = mock_batches_config_instance
# Test the handler
result = AnthropicPassthroughLoggingHandler.batch_creation_handler(
httpx_response=mock_httpx_response,
@ -432,7 +539,7 @@ class TestAnthropicBatchPassthroughCostTracking:
cache_hit=False,
request_body=mock_request_body,
)
# Verify the result
assert result is not None
assert "result" in result
@ -442,33 +549,33 @@ class TestAnthropicBatchPassthroughCostTracking:
assert result["kwargs"]["batch_id"] == "msgbatch_01Wj7gkQk7gn4MpAKR8ZEDU2"
assert result["kwargs"]["batch_job_state"] == "in_progress"
assert "unified_object_id" in result["kwargs"]
# Verify batch was stored
mock_store_batch.assert_called_once()
call_kwargs = mock_store_batch.call_args[1]
assert call_kwargs["model_object_id"] == "msgbatch_01Wj7gkQk7gn4MpAKR8ZEDU2"
assert call_kwargs["batch_object"].status == "validating"
# Verify the response object
assert result["result"].model == "claude-sonnet-4-5-20250929"
assert result["result"].object == "batch"
@patch('litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler._store_batch_managed_object')
@patch('litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler.get_actual_model_id_from_router')
@patch(
"litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler._store_batch_managed_object"
)
@patch(
"litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler.get_actual_model_id_from_router"
)
def test_batch_creation_handler_model_extraction_from_nested_request(
self,
mock_get_model_id,
mock_store_batch,
mock_httpx_response,
mock_logging_obj
self, mock_get_model_id, mock_store_batch, mock_httpx_response, mock_logging_obj
):
"""Test that model is correctly extracted from nested request structure"""
from litellm.llms.anthropic.batches.transformation import AnthropicBatchesConfig
from litellm.types.utils import LiteLLMBatch
# Setup mocks
mock_get_model_id.return_value = "claude-sonnet-4-5-20250929"
mock_batch_response = LiteLLMBatch(
id="msgbatch_123",
object="batch",
@ -479,8 +586,12 @@ class TestAnthropicBatchPassthroughCostTracking:
created_at=1704067200,
request_counts={"total": 1, "completed": 0, "failed": 0},
)
with patch.object(AnthropicBatchesConfig, 'transform_retrieve_batch_response', return_value=mock_batch_response):
with patch.object(
AnthropicBatchesConfig,
"transform_retrieve_batch_response",
return_value=mock_batch_response,
):
# Request body with nested model in requests[0].params.model
request_body = {
"requests": [
@ -488,12 +599,12 @@ class TestAnthropicBatchPassthroughCostTracking:
"custom_id": "test-1",
"params": {
"model": "claude-sonnet-4-5-20250929",
"messages": [{"role": "user", "content": "test"}]
}
"messages": [{"role": "user", "content": "test"}],
},
}
]
}
result = AnthropicPassthroughLoggingHandler.batch_creation_handler(
httpx_response=mock_httpx_response,
logging_obj=mock_logging_obj,
@ -504,26 +615,28 @@ class TestAnthropicBatchPassthroughCostTracking:
cache_hit=False,
request_body=request_body,
)
# Verify model was extracted correctly
assert result["kwargs"]["model"] == "claude-sonnet-4-5-20250929"
@patch('litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler.get_actual_model_id_from_router')
@patch(
"litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler.AnthropicPassthroughLoggingHandler.get_actual_model_id_from_router"
)
def test_batch_creation_handler_model_prefix_when_not_in_router(
self,
mock_get_model_id,
mock_httpx_response,
mock_logging_obj,
mock_request_body
mock_request_body,
):
"""Test that model gets 'anthropic/' prefix when not found in router"""
from litellm.llms.anthropic.batches.transformation import AnthropicBatchesConfig
from litellm.types.utils import LiteLLMBatch
import base64
# Model not in router - returns same model name
mock_get_model_id.return_value = "claude-sonnet-4-5-20250929"
mock_batch_response = LiteLLMBatch(
id="msgbatch_123",
object="batch",
@ -534,9 +647,15 @@ class TestAnthropicBatchPassthroughCostTracking:
created_at=1704067200,
request_counts={"total": 1, "completed": 0, "failed": 0},
)
with patch.object(AnthropicBatchesConfig, 'transform_retrieve_batch_response', return_value=mock_batch_response):
with patch.object(AnthropicPassthroughLoggingHandler, '_store_batch_managed_object'):
with patch.object(
AnthropicBatchesConfig,
"transform_retrieve_batch_response",
return_value=mock_batch_response,
):
with patch.object(
AnthropicPassthroughLoggingHandler, "_store_batch_managed_object"
):
result = AnthropicPassthroughLoggingHandler.batch_creation_handler(
httpx_response=mock_httpx_response,
logging_obj=mock_logging_obj,
@ -547,22 +666,23 @@ class TestAnthropicBatchPassthroughCostTracking:
cache_hit=False,
request_body=mock_request_body,
)
# Verify unified_object_id contains anthropic/ prefix
unified_object_id = result["kwargs"]["unified_object_id"]
decoded = base64.urlsafe_b64decode(unified_object_id + "==").decode()
assert "anthropic/claude-sonnet-4-5-20250929" in decoded or "claude-sonnet-4-5-20250929" in decoded
assert (
"anthropic/claude-sonnet-4-5-20250929" in decoded
or "claude-sonnet-4-5-20250929" in decoded
)
def test_batch_creation_handler_failure_status_code(
self,
mock_logging_obj,
mock_request_body
self, mock_logging_obj, mock_request_body
):
"""Test batch creation handler with non-200 status code"""
mock_response = MagicMock()
mock_response.status_code = 400
mock_response.json.return_value = {"error": "Bad request"}
result = AnthropicPassthroughLoggingHandler.batch_creation_handler(
httpx_response=mock_response,
logging_obj=mock_logging_obj,
@ -573,26 +693,24 @@ class TestAnthropicBatchPassthroughCostTracking:
cache_hit=False,
request_body=mock_request_body,
)
# Verify error response
assert result is not None
assert result["kwargs"]["batch_job_state"] == "failed"
assert result["kwargs"]["response_cost"] == 0.0
@patch('litellm.proxy.proxy_server.proxy_logging_obj')
@patch("litellm.proxy.proxy_server.proxy_logging_obj")
def test_store_batch_managed_object_success(
self,
mock_proxy_logging_obj,
mock_logging_obj
self, mock_proxy_logging_obj, mock_logging_obj
):
"""Test storing batch managed object"""
from litellm.types.utils import LiteLLMBatch
# Setup mocks
mock_managed_files_hook = MagicMock()
mock_managed_files_hook.store_unified_object_id = AsyncMock()
mock_proxy_logging_obj.get_proxy_hook.return_value = mock_managed_files_hook
batch_object = LiteLLMBatch(
id="msgbatch_123",
object="batch",
@ -603,15 +721,17 @@ class TestAnthropicBatchPassthroughCostTracking:
created_at=1704067200,
request_counts={"total": 1, "completed": 0, "failed": 0},
)
with patch('asyncio.create_task'):
with patch("asyncio.create_task"):
AnthropicPassthroughLoggingHandler._store_batch_managed_object(
unified_object_id="test-unified-id",
batch_object=batch_object,
model_object_id="msgbatch_123",
logging_obj=mock_logging_obj,
user_id="test-user"
user_id="test-user",
)
# Verify managed files hook was called
mock_proxy_logging_obj.get_proxy_hook.assert_called_once_with("managed_files")
mock_proxy_logging_obj.get_proxy_hook.assert_called_once_with(
"managed_files"
)