fix(prometheus): keep a caller's own Anthropic OAuth token failure out of the deployment metrics

A provider 401 on an Anthropic OAuth token the caller forwarded through the proxy is the caller's credential
failing, so it stays out of the deployment failure and outage series like a caller-supplied api_key. Adds
integration cells for an Anthropic deployment's own key, the forwarded caller OAuth token and a direct SDK
call that brings its own key

litellm/integrations/prometheus.py

@@ -340,12 +340,16 @@ def _logged_fields(value: object) -> Mapping[str, object]:
 
 def _used_a_caller_credential(logged_call: Mapping[str, object]) -> bool:
     """True when the provider was called with a credential that belongs to the caller rather than the
-    deployment: an api_key the caller sent with the request, or the caller's own GitHub Copilot connection."""
+    deployment: an api_key the caller sent with the request, the caller's own Anthropic OAuth token, or the
+    caller's own GitHub Copilot connection."""
     litellm_params: Final = _logged_fields(logged_call.get("litellm_params"))
     credential_fields: Final = _logged_fields(litellm_params.get("proxy_server_request")).get("credential_fields")
+    logged_metadata: Final = _logged_fields(_logged_fields(logged_call.get("standard_logging_object")).get("metadata"))
     return (
-        isinstance(credential_fields, tuple) and "api_key" in credential_fields
-    ) or github_copilot_user_session_from(litellm_params) is not None
+        (isinstance(credential_fields, tuple) and "api_key" in credential_fields)
+        or logged_metadata.get("used_client_oauth_token") is True
+        or github_copilot_user_session_from(litellm_params) is not None
+    )
 
 
 def _is_deployment_failure(exception: openai.APIError, logged_call: Mapping[str, object]) -> bool:

tests/integration/_support/copilot_session_prometheus_worker.py

@@ -1,6 +1,7 @@
-"""One SDK process for the per-user GitHub Copilot metric cells: a provider 401 on a caller's own Copilot session
-and one on the deployment's own key, then a report of the deployment series on that process's default registry.
-Run as a script by ``run_copilot_session_worker`` so each process registers the Prometheus metrics exactly once."""
+"""One SDK process for the SDK provider-401 metric cells: a provider 401 on a caller's own Copilot session, one on
+the router deployment's own key and one on a direct call's own key, then a report of the deployment series on that
+process's default registry. Run as a script by ``run_copilot_session_worker`` so each process registers the
+Prometheus metrics exactly once."""
 
 from __future__ import annotations
 
@@ -22,6 +23,8 @@ COPILOT_DEPLOYMENT: Final = "copilot-per-user-deployment"
 OPERATOR_DEPLOYMENT: Final = "operator-key-deployment"
 CALLER_SESSION_TOKEN: Final = "caller-copilot-session-token"
 OPERATOR_KEY: Final = "rotated-operator-key"
+DIRECT_CALL: Final = "direct-sdk-call"
+DIRECT_KEY: Final = "rotated-direct-call-key"
 FAILURES: Final = "litellm_deployment_failure_responses_total"
 TOTALS: Final = "litellm_deployment_total_requests_total"
 LLM_FAILED: Final = "litellm_llm_api_failed_requests_metric_total"
@@ -40,17 +43,28 @@ class Outcome:
 class Series:
     metric: str
     model_id: str
+    api_base: str
     value: float
 
 
 @dataclass(frozen=True, slots=True)
 class WorkerReport:
+    api_base: str
     outcomes: tuple[Outcome, ...]
     series: tuple[Series, ...]
 
     def values(self, metric: str, model_id: str) -> tuple[float, ...]:
         return tuple(s.value for s in self.series if s.metric == metric and s.model_id == model_id)
 
+    def on_the_endpoint(self, metric: str, model_id: str) -> tuple[float, ...]:
+        return tuple(
+            s.value for s in self.series if (s.metric, s.model_id, s.api_base) == (metric, model_id, self.api_base)
+        )
+
+    def outside_the_router(self, metric: str) -> tuple[float, ...]:
+        router_ids: Final = (COPILOT_DEPLOYMENT, OPERATOR_DEPLOYMENT)
+        return tuple(s.value for s in self.series if s.metric == metric and s.model_id not in router_ids)
+
 
 _REPORT: Final = TypeAdapter(WorkerReport)
 
@@ -69,7 +83,12 @@ def run_copilot_session_worker(api_base: str) -> WorkerReport:
 
 def _deployment_series() -> tuple[Series, ...]:
     return tuple(
-        Series(metric=sample.name, model_id=str(sample.labels.get("model_id", "")), value=float(sample.value))
+        Series(
+            metric=sample.name,
+            model_id=str(sample.labels.get("model_id", "")),
+            api_base=str(sample.labels.get("api_base", "")),
+            value=float(sample.value),
+        )
         for family in REGISTRY.collect()
         for sample in family.samples
         if sample.name in DEPLOYMENT_SERIES
@@ -95,6 +114,7 @@ def _router(api_base: str) -> litellm.Router:
             },
         ],
         num_retries=0,
+        disable_cooldowns=True,
     )
 
 
@@ -106,6 +126,19 @@ async def _attempt(router: litellm.Router, model: str, **extra: object) -> Outco
     return Outcome(deployment=model, status=None)
 
 
+async def _direct_attempt(api_base: str) -> Outcome:
+    try:
+        await litellm.acompletion(
+            model="openai/gpt-5.6",
+            api_base=api_base,
+            api_key=DIRECT_KEY,
+            messages=[{"role": "user", "content": f"{DIRECT_CALL} worker"}],
+        )
+    except openai.APIStatusError as error:
+        return Outcome(deployment=DIRECT_CALL, status=error.status_code)
+    return Outcome(deployment=DIRECT_CALL, status=None)
+
+
 async def main(api_base: str) -> None:
     litellm.callbacks = [PrometheusLogger()]  # test-quality-ok: this worker process exits right after the report
     session: Final = GithubCopilotUserSession(token=CALLER_SESSION_TOKEN, api_base=api_base)
@@ -113,8 +146,9 @@ async def main(api_base: str) -> None:
     outcomes: Final = (
         await _attempt(router, COPILOT_DEPLOYMENT, github_copilot_user_session=session),
         await _attempt(router, OPERATOR_DEPLOYMENT),
+        await _direct_attempt(api_base),
     )
-    report: Final = WorkerReport(outcomes=outcomes, series=_deployment_series())
+    report: Final = WorkerReport(api_base=api_base, outcomes=outcomes, series=_deployment_series())
     sys.stdout.write(json.dumps(_REPORT.dump_python(report)) + "\n")
 
 

tests/integration/observability/test_prometheus_provider_auth_failure.py

@@ -31,6 +31,7 @@ RATE_LIMITS: Final = "provider-rate-limits"
 RECOVERS: Final = "provider-rejects-once"
 HEALTHY: Final = "provider-healthy"
 AGENT_365_GUARDRAIL: Final = f"agent-365-{RUN}"
+CALLER_OAUTH_TOKEN: Final = f"sk-ant-oat01-caller-{RUN}"
 
 FAILURES: Final = "litellm_deployment_failure_responses_total"
 TOTALS: Final = "litellm_deployment_total_requests_total"
@@ -123,6 +124,18 @@ def deployment(cell: str, upstream_model: str, api_base: str) -> dict[str, JsonV
     }
 
 
+def anthropic_deployment(cell: str, api_base: str) -> dict[str, JsonValue]:
+    return {
+        "model_name": name(cell),
+        "litellm_params": {
+            "model": f"anthropic/{REJECTS}",
+            "api_key": f"sk-ant-api03-deployment-credential-{cell}",
+            "api_base": api_base,
+        },
+        "model_info": {"id": identity(cell)},
+    }
+
+
 def config(directory: Path, upstream: Wire) -> Path:
     api_base: Final = f"{upstream.url}/v1"
     configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
@@ -134,6 +147,8 @@ def config(directory: Path, upstream: Wire) -> Path:
         deployment("key-not-valid", KEY_NOT_VALID, api_base),
         deployment("rate-limited", RATE_LIMITS, api_base),
         deployment("recovers", RECOVERS, api_base),
+        anthropic_deployment("anthropic-own-key", upstream.url),
+        anthropic_deployment("caller-oauth", upstream.url),
         {
             "model_name": name("caller-credential"),
             "litellm_params": {
@@ -384,6 +399,45 @@ def test_a_provider_401_on_an_api_key_the_caller_sent_stays_out_of_the_deploymen
     assert deployment_series(samples) - before == frozenset(), recorded(samples, requested_model=name("caller-api-key"))
 
 
+def anthropic_messages(gateway: Gateway, cell: str, key: str, headers: dict[str, str]) -> httpx.Response:
+    body: Final[dict[str, JsonValue]] = {"model": name(cell), "max_tokens": 16, "messages": messages(cell)}
+    return gateway.request("POST", "/v1/messages", body, key=key, headers=headers)
+
+
+def test_a_provider_401_on_an_anthropic_deployments_own_key_is_recorded(rig: Rig) -> None:
+    with rig.gateway.scenario() as scenario:
+        key: Final = scenario.key(models=[name("anthropic-own-key")])
+        rig.upstream.drain()
+        response: Final = anthropic_messages(rig.gateway, "anthropic-own-key", key, {})
+        assert response.status_code == 401, response.text
+        assert [request.headers.get("x-api-key") for request in rig.upstream.drain()] == [
+            "sk-ant-api03-deployment-credential-anthropic-own-key"
+        ]
+        assert_recorded_as_a_deployment_failure(rig, "anthropic-own-key", "401")
+
+
+def test_a_provider_401_on_an_anthropic_oauth_token_the_caller_sent_stays_out_of_the_deployment_metrics(
+    rig: Rig,
+) -> None:
+    with rig.gateway.scenario() as scenario:
+        key: Final = scenario.key(models=[name("caller-oauth")])
+        rig.upstream.drain()
+        response: Final = anthropic_messages(
+            rig.gateway,
+            "caller-oauth",
+            CALLER_OAUTH_TOKEN,
+            {"x-litellm-api-key": key},
+        )
+        assert response.status_code == 401, response.text
+        assert [request.headers.get("authorization") for request in rig.upstream.drain()] == [
+            f"Bearer {CALLER_OAUTH_TOKEN}"
+        ]
+        samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("caller-oauth"))
+        assert {metric: values(samples, metric, model_id=identity("caller-oauth")) for metric in DEPLOYMENT_SERIES} == {
+            metric: [] for metric in DEPLOYMENT_SERIES
+        }, recorded(samples, requested_model=name("caller-oauth"))
+
+
 def _gated_key(scenario: Scenario, cell: str) -> str | None:
     if cell == "invalid-key":
         return f"sk-not-a-virtual-key-{uuid.uuid4().hex}"

tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py

@@ -1,6 +1,6 @@
 """A provider 401 on a caller's own per-user GitHub Copilot session is the caller's credential failing, so the
-shared Copilot deployment keeps no failure, request or outage series for it, while a provider 401 on the
-deployment's own key in the same process is counted against that deployment."""
+shared Copilot deployment keeps no failure, request or outage series for it, while a provider 401 on the router
+deployment's own key, or on the key a direct SDK call brings, is counted as a failed request."""
 
 from __future__ import annotations
 
@@ -13,6 +13,8 @@ from integration._support.copilot_session_prometheus_worker import (
     CALLER_SESSION_TOKEN,
     COPILOT_DEPLOYMENT,
     DEPLOYMENT_SERIES,
+    DIRECT_CALL,
+    DIRECT_KEY,
     FAILURES,
     LLM_FAILED,
     OPERATOR_DEPLOYMENT,
@@ -47,10 +49,12 @@ def test_both_calls_reach_the_provider_with_their_own_credential_and_fail_with_i
     assert report.outcomes == (
         Outcome(deployment=COPILOT_DEPLOYMENT, status=401),
         Outcome(deployment=OPERATOR_DEPLOYMENT, status=401),
+        Outcome(deployment=DIRECT_CALL, status=401),
     )
     assert [r.headers.get("authorization") for r in received] == [
         f"Bearer {CALLER_SESSION_TOKEN}",
         f"Bearer {OPERATOR_KEY}",
+        f"Bearer {DIRECT_KEY}",
     ]
 
 
@@ -68,4 +72,13 @@ def test_a_provider_401_on_the_deployments_own_key_counts_as_a_deployment_failur
     assert report.values(FAILURES, OPERATOR_DEPLOYMENT) == (1.0,)
     assert report.values(TOTALS, OPERATOR_DEPLOYMENT) == (1.0,)
     assert report.values(LLM_FAILED, OPERATOR_DEPLOYMENT) == (1.0,)
-    assert report.values(STATE, OPERATOR_DEPLOYMENT) == (1.0,)
+    assert report.on_the_endpoint(STATE, OPERATOR_DEPLOYMENT) == (1.0,)
+
+
+def test_a_provider_401_on_the_key_a_direct_sdk_call_brings_is_counted_as_a_failed_request(
+    ran: tuple[WorkerReport, tuple[Request, ...]],
+) -> None:
+    report, _ = ran
+    assert report.outside_the_router(FAILURES) == (1.0,)
+    assert report.outside_the_router(TOTALS) == (1.0,)
+    assert report.outside_the_router(LLM_FAILED) == (1.0,)

tests/unit/integrations/test_prometheus_invalid_key_filtering.py

@@ -176,34 +176,40 @@ class TestSkipMetricsValidation:
         assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected
 
     @pytest.mark.parametrize(
-        "litellm_params,expected",
+        "logged_call,expected",
         [
-            ({"proxy_server_request": {"credential_fields": ("api_key",)}}, True),
+            ({"litellm_params": {"proxy_server_request": {"credential_fields": ("api_key",)}}}, True),
             (
                 {
-                    "github_copilot_user_session": GithubCopilotUserSession(
-                        token="caller-session-token", api_base="https://api.githubcopilot.com"
-                    )
+                    "litellm_params": {
+                        "github_copilot_user_session": GithubCopilotUserSession(
+                            token="caller-session-token", api_base="https://api.githubcopilot.com"
+                        )
+                    }
                 },
                 True,
             ),
-            ({"proxy_server_request": {"credential_fields": ("headers",)}}, False),
-            ({"proxy_server_request": {"credential_fields": "api_key"}}, False),
+            ({"standard_logging_object": {"metadata": {"used_client_oauth_token": True}}}, True),
+            ({"litellm_params": {"proxy_server_request": {"credential_fields": ("headers",)}}}, False),
+            ({"litellm_params": {"proxy_server_request": {"credential_fields": "api_key"}}}, False),
+            ({"standard_logging_object": {"metadata": {"used_client_oauth_token": False}}}, False),
         ],
         ids=[
             "caller-supplied-api-key",
             "caller-copilot-session",
+            "caller-anthropic-oauth-token",
             "caller-supplied-headers-only",
             "malformed-credential-fields",
+            "deployment-anthropic-key",
         ],
     )
-    def test_skip_a_provider_401_on_a_credential_the_caller_owns(self, prometheus_logger, litellm_params, expected):
+    def test_skip_a_provider_401_on_a_credential_the_caller_owns(self, prometheus_logger, logged_call, expected):
         exception = litellm.AuthenticationError(
             message="Incorrect API key provided: sk-calle*****",
             llm_provider="openai",
             model="gpt-4o-mini",
         )
-        kwargs = {"exception": exception, "litellm_params": litellm_params}
+        kwargs = {"exception": exception, **logged_call}
         assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected
 
     def test_no_skip_for_valid_request(self, prometheus_logger):