diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 69f43200fef..99c41309363 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -340,12 +340,16 @@ def _logged_fields(value: object) -> Mapping[str, object]: def _used_a_caller_credential(logged_call: Mapping[str, object]) -> bool: """True when the provider was called with a credential that belongs to the caller rather than the - deployment: an api_key the caller sent with the request, or the caller's own GitHub Copilot connection.""" + deployment: an api_key the caller sent with the request, the caller's own Anthropic OAuth token, or the + caller's own GitHub Copilot connection.""" litellm_params: Final = _logged_fields(logged_call.get("litellm_params")) credential_fields: Final = _logged_fields(litellm_params.get("proxy_server_request")).get("credential_fields") + logged_metadata: Final = _logged_fields(_logged_fields(logged_call.get("standard_logging_object")).get("metadata")) return ( - isinstance(credential_fields, tuple) and "api_key" in credential_fields - ) or github_copilot_user_session_from(litellm_params) is not None + (isinstance(credential_fields, tuple) and "api_key" in credential_fields) + or logged_metadata.get("used_client_oauth_token") is True + or github_copilot_user_session_from(litellm_params) is not None + ) def _is_deployment_failure(exception: openai.APIError, logged_call: Mapping[str, object]) -> bool: diff --git a/tests/integration/_support/copilot_session_prometheus_worker.py b/tests/integration/_support/copilot_session_prometheus_worker.py index f1e45788561..ad4213015a4 100644 --- a/tests/integration/_support/copilot_session_prometheus_worker.py +++ b/tests/integration/_support/copilot_session_prometheus_worker.py @@ -1,6 +1,7 @@ -"""One SDK process for the per-user GitHub Copilot metric cells: a provider 401 on a caller's own Copilot session -and one on the deployment's own key, then a report of the deployment series on that process's default registry. -Run as a script by ``run_copilot_session_worker`` so each process registers the Prometheus metrics exactly once.""" +"""One SDK process for the SDK provider-401 metric cells: a provider 401 on a caller's own Copilot session, one on +the router deployment's own key and one on a direct call's own key, then a report of the deployment series on that +process's default registry. Run as a script by ``run_copilot_session_worker`` so each process registers the +Prometheus metrics exactly once.""" from __future__ import annotations @@ -22,6 +23,8 @@ COPILOT_DEPLOYMENT: Final = "copilot-per-user-deployment" OPERATOR_DEPLOYMENT: Final = "operator-key-deployment" CALLER_SESSION_TOKEN: Final = "caller-copilot-session-token" OPERATOR_KEY: Final = "rotated-operator-key" +DIRECT_CALL: Final = "direct-sdk-call" +DIRECT_KEY: Final = "rotated-direct-call-key" FAILURES: Final = "litellm_deployment_failure_responses_total" TOTALS: Final = "litellm_deployment_total_requests_total" LLM_FAILED: Final = "litellm_llm_api_failed_requests_metric_total" @@ -40,17 +43,28 @@ class Outcome: class Series: metric: str model_id: str + api_base: str value: float @dataclass(frozen=True, slots=True) class WorkerReport: + api_base: str outcomes: tuple[Outcome, ...] series: tuple[Series, ...] def values(self, metric: str, model_id: str) -> tuple[float, ...]: return tuple(s.value for s in self.series if s.metric == metric and s.model_id == model_id) + def on_the_endpoint(self, metric: str, model_id: str) -> tuple[float, ...]: + return tuple( + s.value for s in self.series if (s.metric, s.model_id, s.api_base) == (metric, model_id, self.api_base) + ) + + def outside_the_router(self, metric: str) -> tuple[float, ...]: + router_ids: Final = (COPILOT_DEPLOYMENT, OPERATOR_DEPLOYMENT) + return tuple(s.value for s in self.series if s.metric == metric and s.model_id not in router_ids) + _REPORT: Final = TypeAdapter(WorkerReport) @@ -69,7 +83,12 @@ def run_copilot_session_worker(api_base: str) -> WorkerReport: def _deployment_series() -> tuple[Series, ...]: return tuple( - Series(metric=sample.name, model_id=str(sample.labels.get("model_id", "")), value=float(sample.value)) + Series( + metric=sample.name, + model_id=str(sample.labels.get("model_id", "")), + api_base=str(sample.labels.get("api_base", "")), + value=float(sample.value), + ) for family in REGISTRY.collect() for sample in family.samples if sample.name in DEPLOYMENT_SERIES @@ -95,6 +114,7 @@ def _router(api_base: str) -> litellm.Router: }, ], num_retries=0, + disable_cooldowns=True, ) @@ -106,6 +126,19 @@ async def _attempt(router: litellm.Router, model: str, **extra: object) -> Outco return Outcome(deployment=model, status=None) +async def _direct_attempt(api_base: str) -> Outcome: + try: + await litellm.acompletion( + model="openai/gpt-5.6", + api_base=api_base, + api_key=DIRECT_KEY, + messages=[{"role": "user", "content": f"{DIRECT_CALL} worker"}], + ) + except openai.APIStatusError as error: + return Outcome(deployment=DIRECT_CALL, status=error.status_code) + return Outcome(deployment=DIRECT_CALL, status=None) + + async def main(api_base: str) -> None: litellm.callbacks = [PrometheusLogger()] # test-quality-ok: this worker process exits right after the report session: Final = GithubCopilotUserSession(token=CALLER_SESSION_TOKEN, api_base=api_base) @@ -113,8 +146,9 @@ async def main(api_base: str) -> None: outcomes: Final = ( await _attempt(router, COPILOT_DEPLOYMENT, github_copilot_user_session=session), await _attempt(router, OPERATOR_DEPLOYMENT), + await _direct_attempt(api_base), ) - report: Final = WorkerReport(outcomes=outcomes, series=_deployment_series()) + report: Final = WorkerReport(api_base=api_base, outcomes=outcomes, series=_deployment_series()) sys.stdout.write(json.dumps(_REPORT.dump_python(report)) + "\n") diff --git a/tests/integration/observability/test_prometheus_provider_auth_failure.py b/tests/integration/observability/test_prometheus_provider_auth_failure.py index 6096fef6536..fd1ae0ed56c 100644 --- a/tests/integration/observability/test_prometheus_provider_auth_failure.py +++ b/tests/integration/observability/test_prometheus_provider_auth_failure.py @@ -31,6 +31,7 @@ RATE_LIMITS: Final = "provider-rate-limits" RECOVERS: Final = "provider-rejects-once" HEALTHY: Final = "provider-healthy" AGENT_365_GUARDRAIL: Final = f"agent-365-{RUN}" +CALLER_OAUTH_TOKEN: Final = f"sk-ant-oat01-caller-{RUN}" FAILURES: Final = "litellm_deployment_failure_responses_total" TOTALS: Final = "litellm_deployment_total_requests_total" @@ -123,6 +124,18 @@ def deployment(cell: str, upstream_model: str, api_base: str) -> dict[str, JsonV } +def anthropic_deployment(cell: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"anthropic/{REJECTS}", + "api_key": f"sk-ant-api03-deployment-credential-{cell}", + "api_base": api_base, + }, + "model_info": {"id": identity(cell)}, + } + + def config(directory: Path, upstream: Wire) -> Path: api_base: Final = f"{upstream.url}/v1" configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) @@ -134,6 +147,8 @@ def config(directory: Path, upstream: Wire) -> Path: deployment("key-not-valid", KEY_NOT_VALID, api_base), deployment("rate-limited", RATE_LIMITS, api_base), deployment("recovers", RECOVERS, api_base), + anthropic_deployment("anthropic-own-key", upstream.url), + anthropic_deployment("caller-oauth", upstream.url), { "model_name": name("caller-credential"), "litellm_params": { @@ -384,6 +399,45 @@ def test_a_provider_401_on_an_api_key_the_caller_sent_stays_out_of_the_deploymen assert deployment_series(samples) - before == frozenset(), recorded(samples, requested_model=name("caller-api-key")) +def anthropic_messages(gateway: Gateway, cell: str, key: str, headers: dict[str, str]) -> httpx.Response: + body: Final[dict[str, JsonValue]] = {"model": name(cell), "max_tokens": 16, "messages": messages(cell)} + return gateway.request("POST", "/v1/messages", body, key=key, headers=headers) + + +def test_a_provider_401_on_an_anthropic_deployments_own_key_is_recorded(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[name("anthropic-own-key")]) + rig.upstream.drain() + response: Final = anthropic_messages(rig.gateway, "anthropic-own-key", key, {}) + assert response.status_code == 401, response.text + assert [request.headers.get("x-api-key") for request in rig.upstream.drain()] == [ + "sk-ant-api03-deployment-credential-anthropic-own-key" + ] + assert_recorded_as_a_deployment_failure(rig, "anthropic-own-key", "401") + + +def test_a_provider_401_on_an_anthropic_oauth_token_the_caller_sent_stays_out_of_the_deployment_metrics( + rig: Rig, +) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[name("caller-oauth")]) + rig.upstream.drain() + response: Final = anthropic_messages( + rig.gateway, + "caller-oauth", + CALLER_OAUTH_TOKEN, + {"x-litellm-api-key": key}, + ) + assert response.status_code == 401, response.text + assert [request.headers.get("authorization") for request in rig.upstream.drain()] == [ + f"Bearer {CALLER_OAUTH_TOKEN}" + ] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("caller-oauth")) + assert {metric: values(samples, metric, model_id=identity("caller-oauth")) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name("caller-oauth")) + + def _gated_key(scenario: Scenario, cell: str) -> str | None: if cell == "invalid-key": return f"sk-not-a-virtual-key-{uuid.uuid4().hex}" diff --git a/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py b/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py index 4107dffd68f..1225a3a4aee 100644 --- a/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py +++ b/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py @@ -1,6 +1,6 @@ """A provider 401 on a caller's own per-user GitHub Copilot session is the caller's credential failing, so the -shared Copilot deployment keeps no failure, request or outage series for it, while a provider 401 on the -deployment's own key in the same process is counted against that deployment.""" +shared Copilot deployment keeps no failure, request or outage series for it, while a provider 401 on the router +deployment's own key, or on the key a direct SDK call brings, is counted as a failed request.""" from __future__ import annotations @@ -13,6 +13,8 @@ from integration._support.copilot_session_prometheus_worker import ( CALLER_SESSION_TOKEN, COPILOT_DEPLOYMENT, DEPLOYMENT_SERIES, + DIRECT_CALL, + DIRECT_KEY, FAILURES, LLM_FAILED, OPERATOR_DEPLOYMENT, @@ -47,10 +49,12 @@ def test_both_calls_reach_the_provider_with_their_own_credential_and_fail_with_i assert report.outcomes == ( Outcome(deployment=COPILOT_DEPLOYMENT, status=401), Outcome(deployment=OPERATOR_DEPLOYMENT, status=401), + Outcome(deployment=DIRECT_CALL, status=401), ) assert [r.headers.get("authorization") for r in received] == [ f"Bearer {CALLER_SESSION_TOKEN}", f"Bearer {OPERATOR_KEY}", + f"Bearer {DIRECT_KEY}", ] @@ -68,4 +72,13 @@ def test_a_provider_401_on_the_deployments_own_key_counts_as_a_deployment_failur assert report.values(FAILURES, OPERATOR_DEPLOYMENT) == (1.0,) assert report.values(TOTALS, OPERATOR_DEPLOYMENT) == (1.0,) assert report.values(LLM_FAILED, OPERATOR_DEPLOYMENT) == (1.0,) - assert report.values(STATE, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.on_the_endpoint(STATE, OPERATOR_DEPLOYMENT) == (1.0,) + + +def test_a_provider_401_on_the_key_a_direct_sdk_call_brings_is_counted_as_a_failed_request( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert report.outside_the_router(FAILURES) == (1.0,) + assert report.outside_the_router(TOTALS) == (1.0,) + assert report.outside_the_router(LLM_FAILED) == (1.0,) diff --git a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py index 21e6c70e327..cfaaae7724d 100644 --- a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py +++ b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py @@ -176,34 +176,40 @@ class TestSkipMetricsValidation: assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected @pytest.mark.parametrize( - "litellm_params,expected", + "logged_call,expected", [ - ({"proxy_server_request": {"credential_fields": ("api_key",)}}, True), + ({"litellm_params": {"proxy_server_request": {"credential_fields": ("api_key",)}}}, True), ( { - "github_copilot_user_session": GithubCopilotUserSession( - token="caller-session-token", api_base="https://api.githubcopilot.com" - ) + "litellm_params": { + "github_copilot_user_session": GithubCopilotUserSession( + token="caller-session-token", api_base="https://api.githubcopilot.com" + ) + } }, True, ), - ({"proxy_server_request": {"credential_fields": ("headers",)}}, False), - ({"proxy_server_request": {"credential_fields": "api_key"}}, False), + ({"standard_logging_object": {"metadata": {"used_client_oauth_token": True}}}, True), + ({"litellm_params": {"proxy_server_request": {"credential_fields": ("headers",)}}}, False), + ({"litellm_params": {"proxy_server_request": {"credential_fields": "api_key"}}}, False), + ({"standard_logging_object": {"metadata": {"used_client_oauth_token": False}}}, False), ], ids=[ "caller-supplied-api-key", "caller-copilot-session", + "caller-anthropic-oauth-token", "caller-supplied-headers-only", "malformed-credential-fields", + "deployment-anthropic-key", ], ) - def test_skip_a_provider_401_on_a_credential_the_caller_owns(self, prometheus_logger, litellm_params, expected): + def test_skip_a_provider_401_on_a_credential_the_caller_owns(self, prometheus_logger, logged_call, expected): exception = litellm.AuthenticationError( message="Incorrect API key provided: sk-calle*****", llm_provider="openai", model="gpt-4o-mini", ) - kwargs = {"exception": exception, "litellm_params": litellm_params} + kwargs = {"exception": exception, **logged_call} assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected def test_no_skip_for_valid_request(self, prometheus_logger):