From 2db9a2de878dae3d75cdf88872e84254d89a1be2 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:16:10 -0700 Subject: [PATCH] fix(prometheus): skip a 401 raised before the provider is called or at the caller's own endpoint A deployment with no key of its own fails with LiteLLM's local "Missing Anthropic API Key" 401 before any provider call, and a caller who sends aws_bedrock_runtime_endpoint (or another provider endpoint field) can get a 401 from their own gateway. Both were counted against the shared deployment and marked it as a partial outage, where main skipped every 401. Count a 401 only once the provider was called (api_call_start_time is set), and treat caller-sent provider endpoint fields like caller credential fields. --- litellm/integrations/prometheus.py | 30 +++++---- .../test_prometheus_provider_auth_failure.py | 66 ++++++++++++++++++- .../test_prometheus_invalid_key_filtering.py | 33 +++++++++- 3 files changed, 112 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 96ef6e6ae32..ef8678be471 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -379,31 +379,35 @@ _CALLER_CREDENTIAL_FIELDS: Final = frozenset( ) +_CALLER_ENDPOINT_FIELDS: Final = frozenset( + {"aws_bedrock_runtime_endpoint", "aws_sts_endpoint", "deployment_url", "s3_endpoint_url", "sagemaker_base_url"} +) + + _CALLER_TRANSPORT_FIELDS: Final = frozenset( {"api_key", "extra_headers", "headers", "organization", "provider_specific_header"} ) -def _sent_a_provider_credential(caller_body: Mapping[str, object]) -> bool: - """True when the caller's request body, or its ``extra_body``, carries a provider credential field.""" - return not ( - _CALLER_CREDENTIAL_FIELDS.isdisjoint(caller_body) - and _CALLER_CREDENTIAL_FIELDS.isdisjoint(_logged_fields(caller_body.get("extra_body"))) - ) +def _sent_a_provider_credential_or_endpoint(caller_body: Mapping[str, object]) -> bool: + """True when the caller's request body, or its ``extra_body``, carries a provider credential field or a + provider endpoint field.""" + sent: Final = {*caller_body, *_logged_fields(caller_body.get("extra_body"))} + return not (_CALLER_CREDENTIAL_FIELDS.isdisjoint(sent) and _CALLER_ENDPOINT_FIELDS.isdisjoint(sent)) def _used_a_caller_credential(logged_call: Mapping[str, object]) -> bool: - """True when the provider was called with a credential that belongs to the caller rather than the - deployment: an api_key, header, OpenAI organization or other provider credential field the caller sent with the - request, the caller's own Anthropic OAuth token, the caller's own GitHub Copilot connection, or the caller's - identity token that a token-exchange deployment trades in on every call.""" + """True when the provider was called with a credential or at an endpoint that belongs to the caller rather than + the deployment: an api_key, header, OpenAI organization or other provider credential or endpoint field the caller + sent with the request, the caller's own Anthropic OAuth token, the caller's own GitHub Copilot connection, or the + caller's identity token that a token-exchange deployment trades in on every call.""" litellm_params: Final = _logged_fields(logged_call.get("litellm_params")) proxy_server_request: Final = _logged_fields(litellm_params.get("proxy_server_request")) credential_fields: Final = proxy_server_request.get("credential_fields") logged_metadata: Final = _logged_fields(_logged_fields(logged_call.get("standard_logging_object")).get("metadata")) return ( (isinstance(credential_fields, tuple) and any(field in credential_fields for field in _CALLER_TRANSPORT_FIELDS)) - or _sent_a_provider_credential(_logged_fields(proxy_server_request.get("body"))) + or _sent_a_provider_credential_or_endpoint(_logged_fields(proxy_server_request.get("body"))) or bool(litellm_params.get("token_exchange_endpoint")) or logged_metadata.get("used_client_oauth_token") is True or github_copilot_user_session_from(litellm_params) is not None @@ -412,10 +416,12 @@ def _used_a_caller_credential(logged_call: Mapping[str, object]) -> bool: def _is_deployment_failure(exception: openai.APIError, logged_call: Mapping[str, object]) -> bool: """True for an error raised while calling the deployment with its own credential, on a request the proxy - did not reject itself.""" + did not reject itself. An error LiteLLM raises before it calls the provider, such as a missing deployment key, + is not one.""" return not ( isinstance(exception, CallerCredentialAuthenticationError) or bool(logged_call.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)) + or logged_call.get("api_call_start_time") is None or _used_a_caller_credential(logged_call) ) diff --git a/tests/integration/observability/test_prometheus_provider_auth_failure.py b/tests/integration/observability/test_prometheus_provider_auth_failure.py index 48bc5b2f24b..c886cde3a3a 100644 --- a/tests/integration/observability/test_prometheus_provider_auth_failure.py +++ b/tests/integration/observability/test_prometheus_provider_auth_failure.py @@ -47,6 +47,7 @@ CALLER_CREDENTIAL_CELLS: Final = ( ("caller-gigachat-access-token", "gigachat_access_token"), ) TOKEN_EXCHANGE_CELLS: Final = ("token-exchange", "token-exchange-credential") +KEYLESS_CELLS: Final = ("keyless", "messages-keyless") TOKEN_EXCHANGE_SETTINGS: Final = { "token_exchange_endpoint": "https://login.microsoftonline.com/00000000-0000-4000-8000-0000000c0b01/oauth2/v2.0/token", "client_id": "synthetic-obo-client-id", @@ -136,7 +137,7 @@ class Provider: def __call__(self, request: Request) -> Reply: sent: Final = json.loads(request.body) - model: Final = str(sent.get("model") or sent["model_id"]) + model: Final = str(sent.get("model") or sent.get("model_id") or request.target.split("/")[2]) if request.headers.get("x-portkey-api-key") == REVOKED_GATEWAY_KEY: return _error(401, "Invalid gateway API key", "invalid_request_error", "invalid_api_key") if request.headers.get("openai-organization") == UNKNOWN_ORGANIZATION: @@ -184,6 +185,28 @@ def anthropic_deployment(cell: str, api_base: str) -> dict[str, JsonValue]: } +def bedrock_deployment(cell: str, opt_in: dict[str, JsonValue]) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"bedrock/converse/{REJECTS}", + "aws_access_key_id": "AKIADEPLOYMENTCREDENTIAL", + "aws_secret_access_key": f"deployment-credential-{cell}", + "aws_region_name": "us-east-1", + **opt_in, + }, + "model_info": {"id": identity(cell)}, + } + + +def keyless_anthropic_deployment(cell: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": {"model": f"anthropic/{REJECTS}", "api_base": api_base}, + "model_info": {"id": identity(cell)}, + } + + def azure_deployment(cell: str, api_base: str, credential: dict[str, JsonValue]) -> dict[str, JsonValue]: return { "model_name": name(cell), @@ -240,6 +263,11 @@ def config(directory: Path, upstream: Wire) -> Path: deployment("recovers", RECOVERS, api_base), anthropic_deployment("anthropic-own-key", upstream.url), anthropic_deployment("caller-oauth", upstream.url), + *(keyless_anthropic_deployment(cell, upstream.url) for cell in KEYLESS_CELLS), + bedrock_deployment( + "deployment-caller-bedrock-endpoint", + {"configurable_clientside_auth_params": ["aws_bedrock_runtime_endpoint"]}, + ), azure_deployment("azure-own-token", upstream.url, {"azure_ad_token": "deployment-entra-token-azure-own-token"}), *( azure_deployment( @@ -325,7 +353,11 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: wire_server(Provider(released)) as upstream, gateway_from_environment() as shared, owned_proxy( - shared, directory, {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"}, config=config(directory, upstream) + shared, + directory, + {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"}, + config=config(directory, upstream), + remove_environment=("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"), ) as owned, ): yield Rig(owned, upstream) @@ -339,6 +371,7 @@ def opted_in_config(directory: Path, upstream: Wire) -> Path: azure_deployment(caller_azure_cell("proxy-wide", route, inside_extra_body), upstream.url, {}) for route, inside_extra_body in CALLER_AZURE_CELLS ), + bedrock_deployment("proxy-wide-caller-bedrock-endpoint", {}), ] configuration["general_settings"]["allow_client_side_credentials"] = True configuration["litellm_settings"]["callbacks"] = ["prometheus"] @@ -830,6 +863,35 @@ def test_a_token_exchange_deployment_rejecting_a_caller_without_an_identity_toke }, recorded(samples, requested_model=name(cell)) +@pytest.mark.parametrize("cell", KEYLESS_CELLS) +def test_a_missing_deployment_key_reported_before_the_provider_is_called_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, cell) + assert response.status_code == 401, response.text + assert "Missing Anthropic API Key" in response.text, response.text + assert rig.upstream.drain() == () + assert_kept_out_of_the_deployment_metrics(rig, cell) + + +@pytest.mark.parametrize("opt_in", ("deployment", "proxy-wide")) +def test_a_provider_401_at_a_bedrock_endpoint_the_caller_sends_stays_out_of_the_deployment_metrics( + request: pytest.FixtureRequest, opt_in: str +) -> None: + opted: Final[Rig] = request.getfixturevalue("rig" if opt_in == "deployment" else "opted_in") + cell: Final = f"{opt_in}-caller-bedrock-endpoint" + opted.upstream.drain() + response: Final = opted.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), "aws_bedrock_runtime_endpoint": opted.upstream.url}, + ) + assert response.status_code == 401, response.text + assert [reached.target for reached in opted.upstream.drain()] == [f"/model/{REJECTS}/converse"] + assert_kept_out_of_the_deployment_metrics(opted, cell) + + def test_a_provider_401_at_an_api_base_the_caller_sends_counts_against_a_one_off_deployment(rig: Rig) -> None: caller_api_base: Final = f"{rig.upstream.url}/caller-endpoint-{RUN}/v1" rig.upstream.drain() diff --git a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py index 155edbffc48..78c3f82fbc4 100644 --- a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py +++ b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py @@ -7,6 +7,7 @@ the proxy-level failed request counter still records them. """ from collections.abc import Iterator +from datetime import datetime from typing import Final from unittest.mock import Mock, patch @@ -48,6 +49,14 @@ PROVIDER_CREDENTIAL_FIELDS = [ "wx_credentials", "zen_api_key", ] +PROVIDER_ENDPOINT_FIELDS = [ + "aws_bedrock_runtime_endpoint", + "aws_sts_endpoint", + "deployment_url", + "s3_endpoint_url", + "sagemaker_base_url", +] +CALLED_THE_PROVIDER: Final = {"api_call_start_time": datetime(2026, 10, 19, 12, 0, 0)} @pytest.fixture(scope="function") @@ -201,7 +210,7 @@ class TestSkipMetricsValidation: def test_skip_except_for_provider_errors_the_proxy_did_not_raise( self, prometheus_logger, exception, proxy_gate, expected ): - kwargs: Final[dict[str, object]] = {"exception": exception} + kwargs: Final[dict[str, object]] = {"exception": exception, **CALLED_THE_PROVIDER} if proxy_gate: kwargs[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected @@ -280,10 +289,24 @@ class TestSkipMetricsValidation: llm_provider="openai", model="gpt-4o-mini", ) + kwargs: Final = {"exception": exception, **CALLED_THE_PROVIDER, **logged_call} + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected + + @pytest.mark.parametrize( + "logged_call,expected", + [(CALLED_THE_PROVIDER, False), ({}, True), ({"api_call_start_time": None}, True)], + ids=["provider-401", "401-before-the-provider-call", "blank-provider-call-start"], + ) + def test_skip_a_401_raised_before_the_provider_was_called(self, prometheus_logger, logged_call, expected): + exception: Final = litellm.AuthenticationError( + message="Missing Anthropic API Key - A call is being made to anthropic but no key is set", + llm_provider="anthropic", + model="claude-haiku-4-5", + ) kwargs: Final = {"exception": exception, **logged_call} assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected - @pytest.mark.parametrize("field", PROVIDER_CREDENTIAL_FIELDS) + @pytest.mark.parametrize("field", PROVIDER_CREDENTIAL_FIELDS + PROVIDER_ENDPOINT_FIELDS) @pytest.mark.parametrize("inside_extra_body", [False, True], ids=["body", "extra-body"]) def test_skip_a_provider_401_on_a_provider_credential_field_the_caller_sent( self, prometheus_logger, field, inside_extra_body @@ -293,13 +316,15 @@ class TestSkipMetricsValidation: kwargs: Final = { "exception": litellm.AuthenticationError(message="Unauthorized", llm_provider="azure", model="gpt-4o"), "litellm_params": {"proxy_server_request": {"body": body}}, + **CALLED_THE_PROVIDER, } assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is True - @pytest.mark.parametrize("field", PROVIDER_CREDENTIAL_FIELDS) + @pytest.mark.parametrize("field", PROVIDER_CREDENTIAL_FIELDS + PROVIDER_ENDPOINT_FIELDS) def test_count_a_provider_401_on_a_provider_credential_field_the_deployment_owns(self, prometheus_logger, field): kwargs: Final = { "exception": litellm.AuthenticationError(message="Unauthorized", llm_provider="azure", model="gpt-4o"), + **CALLED_THE_PROVIDER, "litellm_params": { field: "deployment-credential", "proxy_server_request": {"body": {"model": "shared-deployment"}}, @@ -320,6 +345,7 @@ class TestSkipMetricsValidation: message="Unauthorized", llm_provider="microsoft_365_copilot", model="copilot" ), "litellm_params": {"token_exchange_endpoint": token_exchange_endpoint, "client_secret": "deployment"}, + **CALLED_THE_PROVIDER, } assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected @@ -447,6 +473,7 @@ def _provider_auth_failure_kwargs() -> dict[str, object]: }, "exception": exception, "litellm_params": {"custom_llm_provider": "openai", "metadata": {}}, + **CALLED_THE_PROVIDER, }