From ffa37d05b7cf16c9874101f3c737da19b4154aca Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sun, 16 Aug 2026 14:28:50 -0400 Subject: [PATCH 01/53] feat(mistral): add zai-glm-5-2 model pricing and metadata --- .../model_prices_and_context_window_backup.json | 14 ++++++++++++++ model_prices_and_context_window.json | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e6c6cab0631..b73feae90d3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29045,6 +29045,20 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/zai-glm-5-2": { + "input_cost_per_token": 1.4e-06, + "litellm_provider": "mistral", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.mistral.ai/models/zai-glm-5-2", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "mistral/magistral-medium-2506": { "deprecation_date": "2025-11-30", "input_cost_per_token": 2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e6c6cab0631..b73feae90d3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29045,6 +29045,20 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/zai-glm-5-2": { + "input_cost_per_token": 1.4e-06, + "litellm_provider": "mistral", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.mistral.ai/models/zai-glm-5-2", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "mistral/magistral-medium-2506": { "deprecation_date": "2025-11-30", "input_cost_per_token": 2e-06, From 367dd537b995d23421fd99bdb1093e443e98911e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:39:15 -0700 Subject: [PATCH 02/53] feat(e2e): move record/replay to the provider edge (LIT-5745) Replaces the test-side fixture transport with an in-process provider-edge HTTP server the proxy's deployments point their api_base at. Record forwards provider calls verbatim and writes them to the bundle; replay answers them from the bundle with zero provider calls while key auth, routing, cost calculation, and spend-log writes still execute against the live proxy and database. Drift comes back as HTTP 599 naming the computed and closest recorded keys. Request headers are never stored and responses are kept byte-identical between modes from the proxy's side of the socket. --- tests/e2e/CLAUDE.md | 12 +- tests/e2e/CONTRIBUTING.md | 10 +- tests/e2e/conftest.py | 14 +- tests/e2e/e2e_config.py | 32 +- tests/e2e/e2e_http.py | 34 + tests/e2e/fixture_bundle.py | 133 +--- tests/e2e/fixture_mode.py | 132 ++++ tests/e2e/fixture_transport.py | 724 ------------------ tests/e2e/provider_edge.py | 546 +++++++++++++ tests/e2e/proxy_client.py | 16 +- .../test_provider_edge_spend_e2e.py | 50 ++ tests/e2e/test_fixture_bundle.py | 46 +- tests/e2e/test_fixture_mode.py | 114 +++ tests/e2e/test_fixture_transport.py | 676 ---------------- tests/e2e/test_provider_edge.py | 492 ++++++++++++ 15 files changed, 1453 insertions(+), 1578 deletions(-) create mode 100644 tests/e2e/fixture_mode.py delete mode 100644 tests/e2e/fixture_transport.py create mode 100644 tests/e2e/provider_edge.py create mode 100644 tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py create mode 100644 tests/e2e/test_fixture_mode.py delete mode 100644 tests/e2e/test_fixture_transport.py create mode 100644 tests/e2e/test_provider_edge.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index d7334552d0c..840a40a54cd 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -73,13 +73,17 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover ## Record and replay fixtures -`E2E_FIXTURE_MODE` selects the transport every client is built on: `live` (the default, and what an unset variable means: nothing changes), `record` (run against the live proxy and write every interaction to a fixture bundle), or `replay` (serve every interaction back from the bundle with no HTTP at all, so a replay run needs no proxy and cannot bill a provider). The seam is `select_transport` in `fixture_transport.py`, applied inside `build_proxy_client`; both transports fulfil the same `Transport` protocol, so no test or client changes shape in any mode +`E2E_FIXTURE_MODE` scopes the proxy's provider-bound traffic: `live` (the default, and what an unset variable means: nothing changes), `record` (the proxy's provider calls are forwarded to the real provider through a local edge server and written to a fixture bundle), or `replay` (the edge answers those calls from the bundle, so the run makes zero provider calls and spends nothing). Test-to-proxy traffic always goes over the wire in every mode: record and replay both need the live proxy and database, because the point is that key auth, routing, cost calculation, and spend-log writes execute for real while only the provider is swapped out. Breaking any of those in the proxy turns a replay run red -A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per transport call in call order (`0000-post-chat-completions.json`). Auth header values and credential request fields (`api_key`, `*_secret_key`, `static_headers`, and the like; the list is `fixture_canonical.py`'s) are redacted on write, and file uploads store a sha256 digest instead of the bytes; response bodies are stored verbatim (a /key/generate response keeps the ephemeral virtual key it minted), which is part of why bundles are gitignored. `fixture_bundle.py` owns the format +The seam is `provider_edge.py`: `start_provider_edge` boots an in-process HTTP server (one shared instance per pytest process, `e2e_config.provider_edge_base` is the accessor) that mounts each supported provider under a path prefix (`EDGE_MOUNTS`: `/openai` -> `https://api.openai.com`, `/anthropic` -> `https://api.anthropic.com`). A test participates by registering its deployment with `api_base=provider_edge_base("openai")` plus the provider's path suffix; `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` is the reference. In live mode the accessor returns None and the deployment defaults to the real provider, so an edge-wired test runs in all three modes unchanged. Non-wired tests hit their providers live in every mode. The edge binds `E2E_PROVIDER_EDGE_BIND_HOST` (default 127.0.0.1) and advertises `E2E_PROVIDER_EDGE_ADVERTISE_HOST` in the api_base it hands out, for proxies running in containers -Replay matches calls per test by canonical key: `fixture_canonical.py` canonicalizes the recorded request (volatile headers and credential fields out, unique markers, generated ids, uuids, and timestamps replaced with fixed placeholders, object keys sorted) and the key is the method, path, and a content hash, so identity survives re-records and machine changes while any real content drift is a `ReplayMiss` that names the computed key, the closest recorded key with its file, and a content diff, and never falls through to a live call. Matching is order-independent across distinct keys (concurrent calls may interleave) and FIFO within one key (a poll loop replays its responses in recorded order); a passed test must also consume its whole recording, or teardown fails it naming a leftover key. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Every rewrite rule lives in `fixture_canonical.py`, so a new volatile header, credential field name, or generated-id shape is one edit there. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live proxy +A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. `fixture_bundle.py` owns the format. Record serves the proxy the same filtered stored response replay will serve later, so the two modes are byte-identical from the proxy's side of the socket -Deliberately not here yet: streaming chunk fidelity (LIT-5742) and scoping record/replay to provider-bound traffic (LIT-5745) +Replay matches calls per test by canonical key: `fixture_canonical.py` canonicalizes the recorded request (volatile headers and credential fields out, unique markers, generated ids, uuids, and timestamps replaced with fixed placeholders, object keys sorted) and the key is the method, edge path, and a content hash, so identity survives re-records and machine changes while any real content drift comes back as an HTTP 599 naming the computed key, the closest recorded key with its file, and a content diff, and never falls through to a live call. Matching is order-independent across distinct keys (concurrent calls may interleave) and FIFO within one key (a retry loop replays its responses in recorded order); a passed test must also consume its whole recording, or teardown fails it naming a leftover key. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Every rewrite rule lives in `fixture_canonical.py`, so a new volatile header, credential field name, or generated-id shape is one edit there. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live providers + +A replayed response carries the recorded provider response id, and `LiteLLM_SpendLogs.request_id` (the table's primary key) is that id, so a replay against a database that still holds the record run's rows silently dedupes its spend inserts and any spend assertion goes red with zero matching rows and nothing in the proxy log. Run both modes with `E2E_RESET_SPEND_LOGS=1` (plus `DATABASE_URL` in the runner env) so each session truncates the table after itself, or replay against a fresh database, which is the CI shape + +Current limits: streaming chunk fidelity is LIT-5742 (a streamed response records as one buffered body), CI wiring is LIT-5748, Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), multipart uploads have per-run random boundaries (the digest changes every run, so they always miss), and deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base) ## Typing diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 67da1be9562..9096050a45a 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -54,14 +54,16 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT ### Record and replay -`E2E_FIXTURE_MODE=record` runs a suite against the live proxy as usual while writing every request/response pair to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`); `E2E_FIXTURE_MODE=replay` then runs the same suite entirely from that bundle, with no proxy traffic and no provider spend; the proxy liveness gate is skipped, so replay runs with no proxy up at all. Unset (or `live`) behaves exactly as before the knob existed +Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop ```bash -E2E_FIXTURE_MODE=record uv run pytest tests/e2e/llm_translation/ -v -E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/llm_translation/ -v +E2E_FIXTURE_MODE=record uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v +E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v ``` -Replay fails hard (`ReplayMiss`) when the tests drift from the recording, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. See `CLAUDE.md` in this directory for the bundle format and the transport seam +One sharp edge: a replayed response reuses the recorded provider response id, and that id is the primary key of `LiteLLM_SpendLogs`, so replaying against a database that still holds the record run's rows silently dedupes the spend writes and a spend assertion fails with zero rows. Run both commands above with `E2E_RESET_SPEND_LOGS=1` (and `DATABASE_URL` set in the pytest env) so each session truncates the spend log table after itself, or point replay at a fresh database + +Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the current limits (streaming, Bedrock, multipart) Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index da2a7da0bfa..dbe2d6e514e 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -23,12 +23,8 @@ import requests from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup -from fixture_transport import ( - fixture_mode_collection_error, - fixture_report_lines, - parse_fixture_mode, - replay_leftover_error, -) +from fixture_mode import fixture_mode_collection_error, fixture_report_lines +from provider_edge import replay_leftover_error from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from proxy_client import ProxyClient, build_proxy_client @@ -114,12 +110,10 @@ def _proxy_fail_reason() -> str | None: def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. Unmarked tests (unit coverage of the harness) don't touch the proxy, so they - run even when none is up. Never skip for a missing proxy. Replay mode serves - every call from the fixture bundle, so it needs no live proxy either.""" + run even when none is up. Never skip for a missing proxy. Replay mode needs + the proxy too: only provider-bound traffic replays from the bundle.""" if item.get_closest_marker("e2e") is None: return - if parse_fixture_mode(FIXTURE_MODE_RAW) == "replay": - return reason = _proxy_fail_reason() if reason is not None: pytest.fail(reason) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index a5c3729f4be..8bf39f6021f 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -13,7 +13,8 @@ from pathlib import Path from dotenv import load_dotenv -from fixture_transport import deterministic_marker, parse_fixture_mode +from fixture_mode import deterministic_marker, parse_fixture_mode +from provider_edge import provider_edge_api_base # Local runs keep provider / DataDog keys in tests/e2e/.env (see CONTRIBUTING.md). # Compose injects them into the proxy container, but pytest on the host does not @@ -92,15 +93,24 @@ PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15")) EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes") -# Record/replay fixture selection (see fixture_transport.py). The raw mode value -# is parsed and validated there; "live" (the default, also for empty values) -# means the harness behaves exactly as before this knob existed. +# Record/replay fixture selection (see fixture_mode.py and provider_edge.py). +# The raw mode value is parsed and validated there; "live" (the default, also +# for empty values) means the harness behaves exactly as before this knob +# existed. FIXTURE_MODE_RAW = os.environ.get("E2E_FIXTURE_MODE", "live") FIXTURE_DIR = Path( os.environ.get("E2E_FIXTURE_DIR", "").strip() or str(Path(__file__).resolve().parent / ".fixtures") ) +# Where the provider-edge server binds, and the host name edge api_base URLs +# advertise to the proxy. They differ when the proxy runs in a container and +# reaches the pytest host via a gateway name like host.docker.internal. +PROVIDER_EDGE_BIND_HOST = os.environ.get("E2E_PROVIDER_EDGE_BIND_HOST", "").strip() or "127.0.0.1" +PROVIDER_EDGE_ADVERTISE_HOST = ( + os.environ.get("E2E_PROVIDER_EDGE_ADVERTISE_HOST", "").strip() or PROVIDER_EDGE_BIND_HOST +) + # Deliberately modest concurrency. The suite shares its proxy with every other # suite in the run, and 750 users at spawn rate 50 saturated the request path hard # enough to distort latency-sensitive neighbours (and to spend real provider money @@ -157,6 +167,20 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str: return f"{base}?toolsets={toolsets}" if toolsets else base +def provider_edge_base(mount: str) -> str | None: + """The api_base an edge-wired deployment should register with, using this + process's fixture-mode and edge-host configuration: None in live mode, the + shared edge server's mount URL in record and replay.""" + return provider_edge_api_base( + mount, + mode_raw=FIXTURE_MODE_RAW, + bundle_dir=FIXTURE_DIR, + bind_host=PROVIDER_EDGE_BIND_HOST, + advertise_host=PROVIDER_EDGE_ADVERTISE_HOST, + forward_timeout=REQUEST_TIMEOUT, + ) + + def unique_marker() -> str: """A short unique token per call/run, so concurrent runs and the shared response cache never collide on prompts, tags, or customer ids. In record diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index cb6fc7a01e5..03f201e946e 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -647,3 +647,37 @@ def download( content_type=_hdr(resp, "content-type"), body=resp.text, ) + + +class RawResponse(BaseModel): + """A verbatim upstream HTTP response for the provider edge (provider_edge.py): + status, lowercased headers, raw bytes. No Result classification because the + edge relays provider errors to the proxy untouched.""" + + status_code: int + headers: dict[str, str] + body: bytes + + +def forward( + method: str, + url: str, + *, + headers: dict[str, str], + body: bytes | None, + timeout: float = 60.0, +) -> RawResponse | NetworkError: + """Relay one provider-bound request verbatim for the provider edge's record + mode. No retries, no redirects, no schema: the proxy owns retry policy and + the recorded bundle must hold exactly what the provider returned.""" + try: + resp = requests.request( + method, url, headers=headers, data=body, timeout=timeout, allow_redirects=False + ) + except requests.RequestException as exc: + return NetworkError(message=str(exc)) + return RawResponse( + status_code=resp.status_code, + headers={name.lower(): value for name, value in resp.headers.items()}, + body=resp.content, + ) diff --git a/tests/e2e/fixture_bundle.py b/tests/e2e/fixture_bundle.py index 615ae8df1a4..6feb40fc8bc 100644 --- a/tests/e2e/fixture_bundle.py +++ b/tests/e2e/fixture_bundle.py @@ -1,17 +1,18 @@ -"""On-disk fixture bundle format for record/replay e2e runs (LIT-5729). +"""On-disk fixture bundle format for record/replay e2e runs (LIT-5729/LIT-5745). A bundle is a directory: one ``manifest.json`` (record timestamp + harness version + format version) plus one subdirectory per test, holding one JSON file -per transport interaction in call order. Bundles older than +per provider-bound interaction in call order. Bundles older than ``MAX_BUNDLE_AGE`` hard-fail replay at collection time (see conftest), so a green replay run can never certify against fixtures that have drifted more than -a week from the live proxy. +a week from the live providers. -This module owns the format only. The transports that produce and consume it -live in fixture_transport.py and the canonical match keys they compute live in -fixture_canonical.py (LIT-5741); streaming chunk fidelity and provider-scoping -are follow-ups (LIT-5742/5745). Every interaction file stores the full redacted -request because replay matches on its canonicalized content. +This module owns the format only. The provider-edge server that produces and +consumes it lives in provider_edge.py (LIT-5745) and the canonical match keys +it computes live in fixture_canonical.py (LIT-5741); streaming chunk fidelity +is a follow-up (LIT-5742). Every interaction file stores the full redacted +request because replay matches on its canonicalized content, and the response +as the raw HTTP status, filtered headers, and base64 body the provider sent. """ from __future__ import annotations @@ -23,29 +24,14 @@ import subprocess from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Annotated, Final, Literal +from typing import Final -from pydantic import BaseModel, Field, JsonValue, TypeAdapter +from pydantic import BaseModel, JsonValue -from e2e_http import ( - BinaryStream, - NetworkError, - ProbeResult, - RateLimitedError, - Result, - StreamingResponse, - Success, - UnauthorizedError, - UnknownApiError, - ValidationError, -) - -BUNDLE_FORMAT_VERSION: Final = 1 +BUNDLE_FORMAT_VERSION: Final = 2 MAX_BUNDLE_AGE: Final = timedelta(days=7) MANIFEST_FILENAME: Final = "manifest.json" -_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) - class Manifest(BaseModel): format_version: int @@ -54,13 +40,14 @@ class Manifest(BaseModel): class RecordedRequest(BaseModel): - """The request as the transport saw it, auth header values and credential - body/form fields redacted. + """The provider-bound request as the edge saw it, headers empty (SDK + telemetry headers vary run to run and auth material never touches disk). Replay matches on the canonical content key fixture_canonical.py computes - over ``method`` (the transport verb, not the HTTP verb), ``path``, and the - canonicalized headers, params, body, form, and file identity. File uploads - store a content digest instead of the bytes.""" + over ``method``, ``path`` (the edge path including the provider mount, + query string excluded), and the canonicalized headers, params, body, form, + and file identity. Non-JSON bodies store a canonicalized content digest + instead of the bytes.""" method: str path: str @@ -73,85 +60,19 @@ class RecordedRequest(BaseModel): file_bytes: int | None = None -class RecordedResult(BaseModel): - """A ``Result[R]`` flattened for disk. ``data`` holds the success payload as - raw JSON; replay re-validates it against the ``response_type`` the caller - passes, exactly like a live response body.""" +class RecordedHttpResponse(BaseModel): + """The provider's raw HTTP response: status, headers minus hop-by-hop and + volatile entries (see provider_edge.py), and the body as base64 so binary + payloads survive JSON.""" - shape: Literal["result"] = "result" - kind: Literal["success", "network", "unauthorized", "rate_limited", "validation", "unknown"] - status_code: int | None = None - data: JsonValue | None = None - message: str | None = None - body: str | None = None - retry_after_seconds: int | None = None - - -class RecordedStreaming(BaseModel): - shape: Literal["streaming"] = "streaming" - payload: StreamingResponse - - -class RecordedBinary(BaseModel): - shape: Literal["binary"] = "binary" - payload: BinaryStream - - -class RecordedProbe(BaseModel): - shape: Literal["probe"] = "probe" - payload: ProbeResult - - -type RecordedResponse = RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe + status_code: int + headers: dict[str, str] + body_b64: str class Interaction(BaseModel): request: RecordedRequest - response: Annotated[ - RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe, - Field(discriminator="shape"), - ] - - -def to_json_value(model: BaseModel) -> JsonValue: - return _JSON.validate_json(model.model_dump_json(by_alias=True)) - - -def from_result[R: BaseModel](result: Result[R]) -> RecordedResult: - match result: - case Success(status_code=status_code, data=data): - return RecordedResult(kind="success", status_code=status_code, data=to_json_value(data)) - case NetworkError(message=message): - return RecordedResult(kind="network", message=message) - case UnauthorizedError(): - return RecordedResult(kind="unauthorized") - case RateLimitedError(retry_after_seconds=retry_after_seconds, body=body): - return RecordedResult(kind="rate_limited", retry_after_seconds=retry_after_seconds, body=body) - case ValidationError(message=message): - return RecordedResult(kind="validation", message=message) - case UnknownApiError(status_code=status_code, body=body): - return RecordedResult(kind="unknown", status_code=status_code, body=body) - - -def to_result[R: BaseModel](recorded: RecordedResult, response_type: type[R]) -> Result[R]: - match recorded.kind: - case "success": - return Success( - status_code=recorded.status_code or 200, - data=response_type.model_validate(recorded.data), - ) - case "network": - return NetworkError(message=recorded.message or "") - case "unauthorized": - return UnauthorizedError() - case "rate_limited": - return RateLimitedError( - retry_after_seconds=recorded.retry_after_seconds, body=recorded.body or "" - ) - case "validation": - return ValidationError(message=recorded.message or "") - case "unknown": - return UnknownApiError(status_code=recorded.status_code or 0, body=recorded.body or "") + response: RecordedHttpResponse def slugify(raw: str, *, limit: int = 60) -> str: @@ -198,7 +119,7 @@ class BundleRecorder: root: Path _ordinals: dict[str, int] = field(default_factory=dict) - def record(self, *, test_key: str, request: RecordedRequest, response: RecordedResponse) -> None: + def record(self, *, test_key: str, request: RecordedRequest, response: RecordedHttpResponse) -> None: slug = slug_for_test(test_key) ordinal = self._ordinals.get(slug, 0) self._ordinals[slug] = ordinal + 1 diff --git a/tests/e2e/fixture_mode.py b/tests/e2e/fixture_mode.py new file mode 100644 index 00000000000..110f44380b4 --- /dev/null +++ b/tests/e2e/fixture_mode.py @@ -0,0 +1,132 @@ +"""Fixture-mode selection and per-test determinism for record/replay e2e runs. + +``E2E_FIXTURE_MODE`` is live (the default; nothing changes), record, or replay. +This module owns everything mode-shaped that is independent of the provider +edge itself: parsing the raw env value, the collection-time gate that aborts a +run whose mode can never work (unknown value, or replay against a missing or +stale bundle), the pytest report-header lines, the running test's node id, and +the deterministic per-test marker that lets a replay run regenerate exactly +the requests the record run sent. The provider-edge server that records and +serves provider traffic lives in provider_edge.py (LIT-5745). +""" + +from __future__ import annotations + +import hashlib +import os +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Final, Literal, assert_never + +from fixture_bundle import ( + FreshBundle, + StaleBundle, + UnreadableBundle, + check_freshness, + format_age, +) + +type FixtureMode = Literal["live", "record", "replay"] + +FIXTURE_MODES: Final[tuple[FixtureMode, ...]] = ("live", "record", "replay") + +SESSION_TEST_KEY: Final = "session" + + +@dataclass(frozen=True, slots=True) +class InvalidFixtureMode: + value: str + + +def parse_fixture_mode(raw: str) -> FixtureMode | InvalidFixtureMode: + normalized = raw.strip().lower() or "live" + match normalized: + case "live" | "record" | "replay": + return normalized + case _: + return InvalidFixtureMode(value=raw) + + +def current_test_key() -> str: + """The pytest node id of the running test, from the PYTEST_CURRENT_TEST env + var pytest maintains (`` (setup|call|teardown)``); ``session`` for + calls outside any test (e.g. session-finish cleanup).""" + raw = os.environ.get("PYTEST_CURRENT_TEST", "") + if not raw: + return SESSION_TEST_KEY + return raw.rsplit(" (", 1)[0] + + +class ReplayMiss(AssertionError): + """Replay had no recorded interaction for a provider call the proxy made. + The suite drifted from the bundle (or the bundle from the suite): re-record.""" + + +_marker_ordinals: Final[dict[str, int]] = {} + + +def deterministic_marker() -> str: + """Stable stand-in for uuid-based unique markers in record and replay modes: + the Nth marker of a test is a pure function of the test's node id and N, so a + replay run regenerates exactly the model names, prompts, and tags the record + run sent and every recorded provider interaction still matches its key.""" + test_key = current_test_key() + ordinal = _marker_ordinals.get(test_key, 0) + _marker_ordinals[test_key] = ordinal + 1 + return hashlib.sha1(f"{test_key}#{ordinal}".encode()).hexdigest()[:12] + + +def fixture_mode_collection_error(mode_raw: str, bundle_dir: Path, *, now: datetime) -> str | None: + """Session-abort reason for a fixture-mode setup that can never work, or None. + Called at collection time (conftest pytest_sessionstart) so a stale or missing + bundle fails the whole run up front, naming the bundle age, instead of failing + every test individually.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode(value=value): + return f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}" + case "live" | "record": + return None + case "replay": + freshness = check_freshness(bundle_dir, now=now) + match freshness: + case FreshBundle(): + return None + case StaleBundle(recorded_at=recorded_at, age=age, limit=limit): + return ( + f"fixture bundle at {bundle_dir} is stale: recorded {recorded_at.isoformat()}, " + f"age {format_age(age)} exceeds the {limit.days}-day limit; " + "re-record with E2E_FIXTURE_MODE=record" + ) + case UnreadableBundle(reason=reason): + return f"E2E_FIXTURE_MODE=replay cannot use bundle at {bundle_dir}: {reason}" + case _: + assert_never(freshness) + case _: + assert_never(mode) + + +def fixture_report_lines(mode_raw: str, bundle_dir: Path, *, now: datetime) -> list[str]: + """pytest report-header lines; empty in live mode so an unset + E2E_FIXTURE_MODE keeps today's output byte-identical.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode() | "live": + return [] + case "record": + return [f"e2e fixture mode: record -> {bundle_dir}"] + case "replay": + freshness = check_freshness(bundle_dir, now=now) + match freshness: + case FreshBundle(manifest=manifest): + return [ + f"e2e fixture mode: replay <- {bundle_dir} " + f"(recorded {manifest.recorded_at.isoformat()}, harness {manifest.harness_version})" + ] + case StaleBundle() | UnreadableBundle(): + return [f"e2e fixture mode: replay <- {bundle_dir}"] + case _: + assert_never(freshness) + case _: + assert_never(mode) diff --git a/tests/e2e/fixture_transport.py b/tests/e2e/fixture_transport.py deleted file mode 100644 index ce4eec701ca..00000000000 --- a/tests/e2e/fixture_transport.py +++ /dev/null @@ -1,724 +0,0 @@ -"""Record/replay transports behind the same ``Transport`` protocol (LIT-5729). - -``RecordingTransport`` decorates the live transport: every call passes through -unchanged and its request/response pair is appended to the fixture bundle. -``ReplayTransport`` implements the protocol from a recorded bundle alone: no -HTTP, no proxy, no provider spend. Because both fulfil ``Transport``, no test -or client changes shape; ``build_proxy_client`` picks the transport from -``E2E_FIXTURE_MODE`` (live | record | replay, default live). - -Replay matches each call by test node id and canonical content key -(fixture_canonical.py, LIT-5741): volatile headers, credential fields, unique -markers, generated ids, and timestamps are canonicalized out before hashing, so -matching is order-independent across distinct keys, FIFO within a key, and a -miss fails hard (``ReplayMiss``) printing the computed key and the closest -recorded key without ever falling through to a live call. Streaming chunk -fidelity is LIT-5742; scoping record/replay to provider-bound traffic is -LIT-5745. -""" - -from __future__ import annotations - -import difflib -import functools -import hashlib -import os -from collections import deque -from dataclasses import dataclass, field -from datetime import datetime -from itertools import islice -from pathlib import Path -from typing import Final, Literal, assert_never - -from pydantic import BaseModel, JsonValue - -from e2e_http import AuthHeaders, BinaryStream, ProbeResult, Result, StreamingResponse -from fixture_bundle import ( - BundleRecorder, - FreshBundle, - Interaction, - LoadedBundle, - RecordedBinary, - RecordedProbe, - RecordedRequest, - RecordedResponse, - RecordedResult, - RecordedStreaming, - StaleBundle, - UnreadableBundle, - UnsafeBundleDir, - check_freshness, - format_age, - from_result, - interaction_filename, - load_bundle, - prepare_bundle, - slug_for_test, - to_json_value, - to_result, -) -from fixture_canonical import CanonicalRequest, canonicalize, is_secret_field -from transport import Transport - -type FixtureMode = Literal["live", "record", "replay"] - -FIXTURE_MODES: Final[tuple[FixtureMode, ...]] = ("live", "record", "replay") - -SESSION_TEST_KEY: Final = "session" - -REDACTED_HEADER_NAMES: Final[frozenset[str]] = frozenset({"authorization", "x-litellm-api-key"}) -REDACTED_VALUE: Final = "" - - -@dataclass(frozen=True, slots=True) -class InvalidFixtureMode: - value: str - - -def parse_fixture_mode(raw: str) -> FixtureMode | InvalidFixtureMode: - normalized = raw.strip().lower() or "live" - match normalized: - case "live" | "record" | "replay": - return normalized - case _: - return InvalidFixtureMode(value=raw) - - -def current_test_key() -> str: - """The pytest node id of the running test, from the PYTEST_CURRENT_TEST env - var pytest maintains (`` (setup|call|teardown)``); ``session`` for - calls outside any test (e.g. session-finish cleanup).""" - raw = os.environ.get("PYTEST_CURRENT_TEST", "") - if not raw: - return SESSION_TEST_KEY - return raw.rsplit(" (", 1)[0] - - -class ReplayMiss(AssertionError): - """Replay had no recorded interaction for a call the suite made. The test - drifted from the bundle (or the bundle from the suite): re-record.""" - - -_marker_ordinals: Final[dict[str, int]] = {} - - -def deterministic_marker() -> str: - """Stable stand-in for uuid-based unique markers in record and replay modes: - the Nth marker of a test is a pure function of the test's node id and N, so a - replay run regenerates exactly the model names, prompts, and tags the record - run sent and every recorded poll response still satisfies its predicate.""" - test_key = current_test_key() - ordinal = _marker_ordinals.get(test_key, 0) - _marker_ordinals[test_key] = ordinal + 1 - return hashlib.sha1(f"{test_key}#{ordinal}".encode()).hexdigest()[:12] - - -def _dump_flat(model: BaseModel | None) -> dict[str, str]: - if model is None: - return {} - dumped: dict[str, object] = model.model_dump(by_alias=True, exclude_none=True) - return {key: str(value) for key, value in dumped.items()} - - -def _redact(headers: dict[str, str]) -> dict[str, str]: - return { - name: REDACTED_VALUE if name.lower() in REDACTED_HEADER_NAMES else value - for name, value in headers.items() - } - - -def _redact_secret_fields(value: JsonValue) -> JsonValue: - match value: - case dict(): - return { - key: REDACTED_VALUE - if is_secret_field(key) and item is not None - else _redact_secret_fields(item) - for key, item in value.items() - } - case list(): - return [_redact_secret_fields(item) for item in value] - case _: - return value - - -def _redact_flat(fields: dict[str, str]) -> dict[str, str]: - return { - key: REDACTED_VALUE if is_secret_field(key) else value for key, value in fields.items() - } - - -def recorded_request( - method: str, - path: str, - *, - headers: BaseModel, - body: BaseModel | None = None, - params: BaseModel | None = None, - form: BaseModel | None = None, - file_name: str | None = None, - file_content: bytes | None = None, -) -> RecordedRequest: - return RecordedRequest( - method=method, - path=path, - headers=_redact(_dump_flat(headers)), - params=_redact_flat(_dump_flat(params)), - body=None if body is None else _redact_secret_fields(to_json_value(body)), - form=None if form is None else _redact_flat(_dump_flat(form)), - file_name=file_name, - file_sha256=None if file_content is None else hashlib.sha256(file_content).hexdigest(), - file_bytes=None if file_content is None else len(file_content), - ) - - -@dataclass(frozen=True, slots=True) -class RecordingTransport: - """Decorator over the live transport: forwards every call and appends the - interaction to the bundle, so a green live run leaves behind exactly the - traffic replay needs.""" - - inner: Transport - recorder: BundleRecorder - - def _record(self, request: RecordedRequest, response: RecordedResponse) -> None: - self.recorder.record(test_key=current_test_key(), request=request, response=response) - - def bearer(self, key: str) -> AuthHeaders: - return self.inner.bearer(key) - - @property - def master(self) -> AuthHeaders: - return self.inner.master - - def post[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - result = self.inner.post(path, headers=headers, json=json, response_type=response_type) - self._record(recorded_request("post", path, headers=headers, body=json), from_result(result)) - return result - - def get[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - params: BaseModel, - response_type: type[R], - timeout: float | None = None, - ) -> Result[R]: - result = self.inner.get( - path, headers=headers, params=params, response_type=response_type, timeout=timeout - ) - self._record(recorded_request("get", path, headers=headers, params=params), from_result(result)) - return result - - def delete[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - response_type: type[R], - params: BaseModel | None = None, - ) -> Result[R]: - result = self.inner.delete( - path, headers=headers, json=json, response_type=response_type, params=params - ) - self._record( - recorded_request("delete", path, headers=headers, body=json, params=params), - from_result(result), - ) - return result - - def patch[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - result = self.inner.patch(path, headers=headers, json=json, response_type=response_type) - self._record(recorded_request("patch", path, headers=headers, body=json), from_result(result)) - return result - - def put[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - result = self.inner.put(path, headers=headers, json=json, response_type=response_type) - self._record(recorded_request("put", path, headers=headers, body=json), from_result(result)) - return result - - def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: - response = self.inner.stream(path, headers=headers, json=json) - self._record( - recorded_request("stream", path, headers=headers, body=json), - RecordedStreaming(payload=response), - ) - return response - - def stream_binary( - self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 - ) -> BinaryStream: - response = self.inner.stream_binary(path, headers=headers, json=json, chunk_size=chunk_size) - self._record( - recorded_request("stream_binary", path, headers=headers, body=json), - RecordedBinary(payload=response), - ) - return response - - def send( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - params: BaseModel | None = None, - stream: bool = False, - ) -> StreamingResponse: - response = self.inner.send(path, headers=headers, json=json, params=params, stream=stream) - self._record( - recorded_request("send", path, headers=headers, body=json, params=params), - RecordedStreaming(payload=response), - ) - return response - - def probe(self, path: str, *, params: BaseModel) -> ProbeResult: - response = self.inner.probe(path, params=params) - self._record( - recorded_request("probe", path, headers=self.master, params=params), - RecordedProbe(payload=response), - ) - return response - - def upload[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - form: BaseModel, - filename: str, - content: bytes, - file_content_type: str = "application/jsonl", - file_field: str = "file", - params: BaseModel | None = None, - response_type: type[R], - ) -> Result[R]: - result = self.inner.upload( - path, - headers=headers, - form=form, - filename=filename, - content=content, - file_content_type=file_content_type, - file_field=file_field, - params=params, - response_type=response_type, - ) - self._record( - recorded_request( - "upload", - path, - headers=headers, - params=params, - form=form, - file_name=filename, - file_content=content, - ), - from_result(result), - ) - return result - - def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: - response = self.inner.download(path, headers=headers) - self._record( - recorded_request("download", path, headers=headers), - RecordedStreaming(payload=response), - ) - return response - - -def _build_pool(recorded: tuple[Interaction, ...]) -> dict[str, deque[Interaction]]: - keys: Final = tuple(canonicalize(interaction.request).key for interaction in recorded) - return { - key: deque( - interaction - for candidate_key, interaction in zip(keys, recorded, strict=True) - if candidate_key == key - ) - for key in dict.fromkeys(keys) - } - - -def _closest_recorded( - canonical: CanonicalRequest, recorded: tuple[Interaction, ...] -) -> tuple[CanonicalRequest, str]: - candidates: Final = tuple(canonicalize(interaction.request) for interaction in recorded) - ratios: Final = tuple( - difflib.SequenceMatcher( - None, f"{canonical.method} {canonical.path}\n{canonical.content}", - f"{candidate.method} {candidate.path}\n{candidate.content}", - ).ratio() - for candidate in candidates - ) - best: Final = max(range(len(candidates)), key=lambda index: ratios[index]) - return candidates[best], interaction_filename(best, recorded[best].request) - - -def _miss_message(test_key: str, slug: str, canonical: CanonicalRequest, bundle: LoadedBundle) -> str: - recorded: Final = bundle.interactions.get(slug, ()) - if not recorded: - return ( - f"replay miss for {test_key}: computed key {canonical.key} but nothing is recorded " - f"under {slug}; re-record with E2E_FIXTURE_MODE=record" - ) - closest, closest_file = _closest_recorded(canonical, recorded) - diff: Final = "\n".join( - islice( - difflib.unified_diff( - closest.pretty_content().splitlines(), - canonical.pretty_content().splitlines(), - fromfile=f"closest recorded ({closest_file})", - tofile="test made", - lineterm="", - ), - 60, - ) - ) - return ( - f"replay miss for {test_key}: no recorded interaction matches key {canonical.key}; " - f"closest recorded key is {closest.key} ({closest_file})\n{diff}\n" - "re-record with E2E_FIXTURE_MODE=record" - ) - - -@dataclass(slots=True) -class ReplaySource: - """One shared pool per test over a loaded bundle, so every client built in - the session consumes the same recorded interactions. Every pool is built - once at construction and per-key consumption is a single atomic deque pop, - so concurrent replay calls never race. Calls match by canonical content - key: order-independent across distinct keys (concurrent tests interleave - calls nondeterministically), FIFO within one key (a poll loop replays its - recorded responses in recorded order).""" - - bundle: LoadedBundle - _pools: dict[str, dict[str, deque[Interaction]]] = field(init=False) - - def __post_init__(self) -> None: - self._pools = { - slug: _build_pool(recorded) for slug, recorded in self.bundle.interactions.items() - } - - def _pool(self, slug: str) -> dict[str, deque[Interaction]]: - return self._pools.get(slug, {}) - - def next_interaction(self, request: RecordedRequest) -> Interaction: - test_key: Final = current_test_key() - slug: Final = slug_for_test(test_key) - pool: Final = self._pool(slug) - canonical: Final = canonicalize(request) - queue: Final = pool.get(canonical.key) - if queue is None: - raise ReplayMiss(_miss_message(test_key, slug, canonical, self.bundle)) - try: - return queue.popleft() - except IndexError: - raise ReplayMiss( - f"replay exhausted for {test_key}: every recorded interaction for key " - f"{canonical.key} is already consumed; re-record with E2E_FIXTURE_MODE=record" - ) from None - - def leftover_error(self, test_key: str) -> str | None: - """Non-None when the test consumed fewer interactions than were recorded, - meaning a passing replay proved less than the bundle claims.""" - slug: Final = slug_for_test(test_key) - recorded: Final = self.bundle.interactions.get(slug, ()) - if not recorded: - return None - leftover: Final = tuple( - interaction for queue in self._pool(slug).values() for interaction in queue - ) - if not leftover: - return None - return ( - f"replay incomplete for {test_key}: {len(leftover)} of {len(recorded)} recorded " - f"interactions never consumed, e.g. {canonicalize(leftover[0].request).key}; " - "re-record with E2E_FIXTURE_MODE=record" - ) - - -def _expect_result(interaction: Interaction) -> RecordedResult: - match interaction.response: - case RecordedResult() as recorded: - return recorded - case RecordedStreaming() | RecordedBinary() | RecordedProbe(): - raise ReplayMiss( - f"recorded {interaction.request.method} {interaction.request.path} is not a typed result" - ) - - -def _expect_streaming(interaction: Interaction) -> StreamingResponse: - match interaction.response: - case RecordedStreaming(payload=payload): - return payload - case RecordedResult() | RecordedBinary() | RecordedProbe(): - raise ReplayMiss( - f"recorded {interaction.request.method} {interaction.request.path} is not a streaming response" - ) - - -@dataclass(frozen=True, slots=True) -class ReplayTransport: - """A ``Transport`` served entirely from a recorded bundle: never opens a - connection, so a replay run cannot bill a provider.""" - - source: ReplaySource - master_key: str - - def bearer(self, key: str) -> AuthHeaders: - return AuthHeaders(authorization=f"Bearer {key}") - - @property - def master(self) -> AuthHeaders: - return self.bearer(self.master_key) - - def post[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction(recorded_request("post", path, headers=headers, body=json)) - ), - response_type, - ) - - def get[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - params: BaseModel, - response_type: type[R], - timeout: float | None = None, - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction(recorded_request("get", path, headers=headers, params=params)) - ), - response_type, - ) - - def delete[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - response_type: type[R], - params: BaseModel | None = None, - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction( - recorded_request("delete", path, headers=headers, body=json, params=params) - ) - ), - response_type, - ) - - def patch[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction(recorded_request("patch", path, headers=headers, body=json)) - ), - response_type, - ) - - def put[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction(recorded_request("put", path, headers=headers, body=json)) - ), - response_type, - ) - - def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: - return _expect_streaming( - self.source.next_interaction(recorded_request("stream", path, headers=headers, body=json)) - ) - - def stream_binary( - self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 - ) -> BinaryStream: - interaction = self.source.next_interaction( - recorded_request("stream_binary", path, headers=headers, body=json) - ) - match interaction.response: - case RecordedBinary(payload=payload): - return payload - case RecordedResult() | RecordedStreaming() | RecordedProbe(): - raise ReplayMiss( - f"recorded stream_binary {interaction.request.path} is not a binary stream" - ) - - def send( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - params: BaseModel | None = None, - stream: bool = False, - ) -> StreamingResponse: - return _expect_streaming( - self.source.next_interaction( - recorded_request("send", path, headers=headers, body=json, params=params) - ) - ) - - def probe(self, path: str, *, params: BaseModel) -> ProbeResult: - interaction = self.source.next_interaction( - recorded_request("probe", path, headers=self.master, params=params) - ) - match interaction.response: - case RecordedProbe(payload=payload): - return payload - case RecordedResult() | RecordedStreaming() | RecordedBinary(): - raise ReplayMiss(f"recorded probe {interaction.request.path} is not a probe result") - - def upload[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - form: BaseModel, - filename: str, - content: bytes, - file_content_type: str = "application/jsonl", - file_field: str = "file", - params: BaseModel | None = None, - response_type: type[R], - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction( - recorded_request( - "upload", - path, - headers=headers, - params=params, - form=form, - file_name=filename, - file_content=content, - ) - ) - ), - response_type, - ) - - def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: - return _expect_streaming( - self.source.next_interaction(recorded_request("download", path, headers=headers)) - ) - - -@functools.lru_cache(maxsize=8) -def _shared_recorder(root: Path) -> BundleRecorder: - prepared = prepare_bundle(root) - if isinstance(prepared, UnsafeBundleDir): - raise ValueError(f"E2E_FIXTURE_DIR {prepared.path} {prepared.reason}") - return prepared - - -@functools.lru_cache(maxsize=8) -def _shared_replay_source(root: Path) -> ReplaySource: - loaded = load_bundle(root) - if isinstance(loaded, UnreadableBundle): - raise ValueError(f"cannot replay from {root}: {loaded.reason}") - return ReplaySource(bundle=loaded) - - -def replay_leftover_error(*, mode_raw: str, bundle_dir: Path, test_key: str) -> str | None: - """Teardown-time completeness check: in replay mode a passed test with - unconsumed recorded interactions must fail instead of passing against a - recording it no longer matches. Inert in every other mode.""" - if parse_fixture_mode(mode_raw) != "replay": - return None - return _shared_replay_source(bundle_dir).leftover_error(test_key) - - -def select_transport( - live: Transport, *, mode_raw: str, bundle_dir: Path, master_key: str -) -> Transport: - """The one seam every client build goes through: wraps (record), replaces - (replay), or passes through (live) the transport per E2E_FIXTURE_MODE. The - recorder and replay cursors are process-wide singletons per bundle dir, so - every client in a session shares one bundle and one recorded sequence.""" - mode = parse_fixture_mode(mode_raw) - match mode: - case InvalidFixtureMode(value=value): - raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") - case "live": - return live - case "record": - return RecordingTransport(inner=live, recorder=_shared_recorder(bundle_dir)) - case "replay": - return ReplayTransport(source=_shared_replay_source(bundle_dir), master_key=master_key) - case _: - assert_never(mode) - - -def fixture_mode_collection_error(mode_raw: str, bundle_dir: Path, *, now: datetime) -> str | None: - """Session-abort reason for a fixture-mode setup that can never work, or None. - Called at collection time (conftest pytest_sessionstart) so a stale or missing - bundle fails the whole run up front, naming the bundle age, instead of failing - every test individually.""" - mode = parse_fixture_mode(mode_raw) - match mode: - case InvalidFixtureMode(value=value): - return f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}" - case "live" | "record": - return None - case "replay": - freshness = check_freshness(bundle_dir, now=now) - match freshness: - case FreshBundle(): - return None - case StaleBundle(recorded_at=recorded_at, age=age, limit=limit): - return ( - f"fixture bundle at {bundle_dir} is stale: recorded {recorded_at.isoformat()}, " - f"age {format_age(age)} exceeds the {limit.days}-day limit; " - "re-record with E2E_FIXTURE_MODE=record" - ) - case UnreadableBundle(reason=reason): - return f"E2E_FIXTURE_MODE=replay cannot use bundle at {bundle_dir}: {reason}" - case _: - assert_never(freshness) - case _: - assert_never(mode) - - -def fixture_report_lines(mode_raw: str, bundle_dir: Path, *, now: datetime) -> list[str]: - """pytest report-header lines; empty in live mode so an unset - E2E_FIXTURE_MODE keeps today's output byte-identical.""" - mode = parse_fixture_mode(mode_raw) - match mode: - case InvalidFixtureMode() | "live": - return [] - case "record": - return [f"e2e fixture mode: record -> {bundle_dir}"] - case "replay": - freshness = check_freshness(bundle_dir, now=now) - match freshness: - case FreshBundle(manifest=manifest): - return [ - f"e2e fixture mode: replay <- {bundle_dir} " - f"(recorded {manifest.recorded_at.isoformat()}, harness {manifest.harness_version})" - ] - case StaleBundle() | UnreadableBundle(): - return [f"e2e fixture mode: replay <- {bundle_dir}"] - case _: - assert_never(freshness) - case _: - assert_never(mode) diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py new file mode 100644 index 00000000000..ab0791e6b74 --- /dev/null +++ b/tests/e2e/provider_edge.py @@ -0,0 +1,546 @@ +"""Provider-edge record/replay server for e2e runs (LIT-5745). + +Record and replay scope to provider-bound traffic only: the proxy boots for +real, tests hit it for real, and only the hop from the proxy to the provider +is recorded or served from a bundle. Suites opt in per deployment by pointing +``litellm_params.api_base`` at ``provider_edge_api_base(mount)``, which is an +in-process HTTP server mounting each supported provider under a path prefix +(``http://127.0.0.1:/openai`` forwards to ``https://api.openai.com``). +In record mode the edge relays each request verbatim, stores the interaction, +and serves the proxy the same filtered response replay will serve later; in +replay mode it serves straight from the bundle and never opens a provider +connection, so a green replay run with a fake provider key proves the entire +proxy pipeline (auth, routing, spend logging) without provider spend. + +Request identity reuses fixture_canonical.py: interactions match by canonical +content key, order-independent across keys and FIFO within one. Edge requests +store no headers at all: SDK telemetry headers vary run to run and credential +headers must never touch disk. An unmatched replay call returns HTTP +``REPLAY_MISS_STATUS`` naming the closest recorded interaction, which the +proxy relays as a provider error the failing test surfaces. + +v1 limits: only the mounts in ``EDGE_MOUNTS`` (SigV4 providers like Bedrock +sign the Host header, so a forwarding edge breaks their signatures), JSON and +opaque single-part bodies (multipart boundaries are random per request), +streaming fidelity is LIT-5742, and CI wiring is LIT-5748. Suites that do not +wire the edge keep hitting providers live in every mode. +""" + +from __future__ import annotations + +import base64 +import difflib +import functools +import hashlib +import threading +from collections import deque +from collections.abc import Mapping +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from itertools import islice +from pathlib import Path +from types import MappingProxyType +from typing import Final, Literal, assert_never +from urllib.parse import parse_qsl, urlsplit + +from pydantic import JsonValue, TypeAdapter + +from e2e_http import NetworkError, RawResponse, forward +from fixture_bundle import ( + BundleRecorder, + Interaction, + LoadedBundle, + RecordedHttpResponse, + RecordedRequest, + UnreadableBundle, + UnsafeBundleDir, + interaction_filename, + load_bundle, + prepare_bundle, + slug_for_test, +) +from fixture_canonical import CanonicalRequest, canonical_string, canonicalize +from fixture_mode import ( + FIXTURE_MODES, + InvalidFixtureMode, + ReplayMiss, + current_test_key, + parse_fixture_mode, +) + +EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( + { + "openai": "https://api.openai.com", + "anthropic": "https://api.anthropic.com", + } +) + +REPLAY_MISS_STATUS: Final = 599 + +_HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + } +) +_REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { + "host", + "content-length", + "accept-encoding", +} +_RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { + "content-encoding", + "content-length", + "set-cookie", +} + +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def _edge_request(method: str, path: str, query: str, body: bytes | None) -> RecordedRequest: + """The identity replay matches on: the edge path (mount included), the query + as params, and the body as parsed JSON, or as a canonicalized content digest + when it is not JSON so opaque uploads still match across runs.""" + params: Final = dict(parse_qsl(query, keep_blank_values=True)) + if not body: + return RecordedRequest(method=method.lower(), path=path, headers={}, params=params) + decoded: Final = body.decode("utf-8", errors="replace") + try: + parsed: Final[JsonValue] = _JSON.validate_json(decoded) + except ValueError: + return RecordedRequest( + method=method.lower(), + path=path, + headers={}, + params=params, + file_sha256=hashlib.sha256(canonical_string(decoded).encode()).hexdigest(), + file_bytes=len(body), + ) + return RecordedRequest(method=method.lower(), path=path, headers={}, params=params, body=parsed) + + +def _build_pool(recorded: tuple[Interaction, ...]) -> dict[str, deque[Interaction]]: + keys: Final = tuple(canonicalize(interaction.request).key for interaction in recorded) + return { + key: deque( + interaction + for candidate_key, interaction in zip(keys, recorded, strict=True) + if candidate_key == key + ) + for key in dict.fromkeys(keys) + } + + +def _closest_recorded( + canonical: CanonicalRequest, recorded: tuple[Interaction, ...] +) -> tuple[CanonicalRequest, str]: + candidates: Final = tuple(canonicalize(interaction.request) for interaction in recorded) + ratios: Final = tuple( + difflib.SequenceMatcher( + None, f"{canonical.method} {canonical.path}\n{canonical.content}", + f"{candidate.method} {candidate.path}\n{candidate.content}", + ).ratio() + for candidate in candidates + ) + best: Final = max(range(len(candidates)), key=lambda index: ratios[index]) + return candidates[best], interaction_filename(best, recorded[best].request) + + +def _miss_message(test_key: str, slug: str, canonical: CanonicalRequest, bundle: LoadedBundle) -> str: + recorded: Final = bundle.interactions.get(slug, ()) + if not recorded: + return ( + f"replay miss for {test_key}: computed key {canonical.key} but nothing is recorded " + f"under {slug}; re-record with E2E_FIXTURE_MODE=record" + ) + closest, closest_file = _closest_recorded(canonical, recorded) + diff: Final = "\n".join( + islice( + difflib.unified_diff( + closest.pretty_content().splitlines(), + canonical.pretty_content().splitlines(), + fromfile=f"closest recorded ({closest_file})", + tofile="test made", + lineterm="", + ), + 60, + ) + ) + return ( + f"replay miss for {test_key}: no recorded interaction matches key {canonical.key}; " + f"closest recorded key is {closest.key} ({closest_file})\n{diff}\n" + "re-record with E2E_FIXTURE_MODE=record" + ) + + +@dataclass(slots=True) +class ReplaySource: + """One shared pool per test over a loaded bundle, so every provider call the + proxy makes in the session consumes from the same recorded interactions. + Every pool is built once at construction and per-key consumption is a single + atomic deque pop, so concurrent replay calls never race. Calls match by + canonical content key: order-independent across distinct keys (concurrent + tests interleave calls nondeterministically), FIFO within one key (a retry + or poll loop replays its recorded responses in recorded order).""" + + bundle: LoadedBundle + _pools: dict[str, dict[str, deque[Interaction]]] = field(init=False) + + def __post_init__(self) -> None: + self._pools = { + slug: _build_pool(recorded) for slug, recorded in self.bundle.interactions.items() + } + + def _pool(self, slug: str) -> dict[str, deque[Interaction]]: + return self._pools.get(slug, {}) + + def next_interaction(self, request: RecordedRequest) -> Interaction: + test_key: Final = current_test_key() + slug: Final = slug_for_test(test_key) + pool: Final = self._pool(slug) + canonical: Final = canonicalize(request) + queue: Final = pool.get(canonical.key) + if queue is None: + raise ReplayMiss(_miss_message(test_key, slug, canonical, self.bundle)) + try: + return queue.popleft() + except IndexError: + raise ReplayMiss( + f"replay exhausted for {test_key}: every recorded interaction for key " + f"{canonical.key} is already consumed; re-record with E2E_FIXTURE_MODE=record" + ) from None + + def leftover_error(self, test_key: str) -> str | None: + """Non-None when the test consumed fewer interactions than were recorded, + meaning a passing replay proved less than the bundle claims.""" + slug: Final = slug_for_test(test_key) + recorded: Final = self.bundle.interactions.get(slug, ()) + if not recorded: + return None + leftover: Final = tuple( + interaction for queue in self._pool(slug).values() for interaction in queue + ) + if not leftover: + return None + return ( + f"replay incomplete for {test_key}: {len(leftover)} of {len(recorded)} recorded " + f"interactions never consumed, e.g. {canonicalize(leftover[0].request).key}; " + "re-record with E2E_FIXTURE_MODE=record" + ) + + +@dataclass(frozen=True, slots=True) +class RecordEdge: + """Record backend: forward to the provider, persist, serve the filtered copy. + The lock serializes recorder writes because the edge server handles requests + on concurrent threads.""" + + recorder: BundleRecorder + lock: threading.Lock + + +@dataclass(frozen=True, slots=True) +class ReplayEdge: + source: ReplaySource + + +type EdgeBackend = RecordEdge | ReplayEdge + + +@dataclass(frozen=True, slots=True) +class EdgeReply: + status_code: int + headers: dict[str, str] + body: bytes + + +def _text_reply(status_code: int, message: str) -> EdgeReply: + return EdgeReply( + status_code=status_code, + headers={"content-type": "text/plain; charset=utf-8"}, + body=message.encode(), + ) + + +def _reply_from_recorded(response: RecordedHttpResponse) -> EdgeReply: + return EdgeReply( + status_code=response.status_code, + headers=dict(response.headers), + body=base64.b64decode(response.body_b64), + ) + + +def _recorded_response(outcome: RawResponse | NetworkError) -> RecordedHttpResponse: + match outcome: + case RawResponse(status_code=status_code, headers=headers, body=body): + return RecordedHttpResponse( + status_code=status_code, + headers={ + name: value + for name, value in headers.items() + if name not in _RESPONSE_DROPPED_HEADERS + }, + body_b64=base64.b64encode(body).decode("ascii"), + ) + case NetworkError(message=message): + return RecordedHttpResponse( + status_code=502, + headers={"content-type": "text/plain; charset=utf-8"}, + body_b64=base64.b64encode( + f"provider edge could not reach the provider: {message}".encode() + ).decode("ascii"), + ) + + +def _upstream_url(upstream_base: str, upstream_path: str, query: str) -> str: + url: Final = f"{upstream_base}/{upstream_path}" + return f"{url}?{query}" if query else url + + +def _handle_record( + backend: RecordEdge, + request: RecordedRequest, + *, + method: str, + url: str, + headers: Mapping[str, str], + body: bytes | None, + timeout: float, +) -> EdgeReply: + forwarded: Final = { + name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS + } + outcome: Final = forward(method, url, headers=forwarded, body=body, timeout=timeout) + response: Final = _recorded_response(outcome) + with backend.lock: + backend.recorder.record(test_key=current_test_key(), request=request, response=response) + return _reply_from_recorded(response) + + +def _handle_replay(source: ReplaySource, request: RecordedRequest) -> EdgeReply: + try: + interaction: Final = source.next_interaction(request) + except ReplayMiss as miss: + return _text_reply(REPLAY_MISS_STATUS, str(miss)) + return _reply_from_recorded(interaction.response) + + +def handle_edge_request( + backend: EdgeBackend, + mounts: Mapping[str, str], + method: str, + raw_path: str, + headers: Mapping[str, str], + body: bytes | None, + *, + timeout: float, +) -> EdgeReply: + """The edge's pure core, one HTTP exchange in and out: resolve the mount + prefix, then record (forward + persist) or replay (serve from the bundle). + Socket-free so unit tests exercise every branch without a server.""" + split: Final = urlsplit(raw_path) + mount, _, upstream_path = split.path.lstrip("/").partition("/") + upstream_base: Final = mounts.get(mount) + if upstream_base is None: + return _text_reply( + 404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}" + ) + request: Final = _edge_request(method, split.path, split.query, body) + match backend: + case RecordEdge(): + return _handle_record( + backend, + request, + method=method, + url=_upstream_url(upstream_base, upstream_path, split.query), + headers=headers, + body=body, + timeout=timeout, + ) + case ReplayEdge(source=source): + return _handle_replay(source, request) + case _: + assert_never(backend) + + +class _EdgeHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + self._handle() + + def do_POST(self) -> None: + self._handle() + + def do_PUT(self) -> None: + self._handle() + + def do_PATCH(self) -> None: + self._handle() + + def do_DELETE(self) -> None: + self._handle() + + def _handle(self) -> None: + edge_server: Final = self.server + assert isinstance(edge_server, _EdgeHTTPServer) + length: Final = int(self.headers.get("content-length") or "0") + body: Final = self.rfile.read(length) if length else None + reply: Final = handle_edge_request( + edge_server.backend, + edge_server.mounts, + self.command, + self.path, + {name.lower(): value for name, value in self.headers.items()}, + body, + timeout=edge_server.forward_timeout, + ) + self.send_response(reply.status_code) + for name, value in reply.headers.items(): + self.send_header(name, value) + self.send_header("content-length", str(len(reply.body))) + self.end_headers() + self.wfile.write(reply.body) + + def log_message(self, format: str, *args: object) -> None: + """Silence the per-request stderr line BaseHTTPRequestHandler emits.""" + + +class _EdgeHTTPServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__( + self, + bind: tuple[str, int], + *, + backend: EdgeBackend, + mounts: Mapping[str, str], + forward_timeout: float, + ) -> None: + super().__init__(bind, _EdgeHandler) + self.backend: Final = backend + self.mounts: Final = mounts + self.forward_timeout: Final = forward_timeout + + +@dataclass(frozen=True, slots=True) +class ProviderEdge: + port: int + advertise_host: str + + def api_base(self, mount: str) -> str: + return f"http://{self.advertise_host}:{self.port}/{mount}" + + +@dataclass(frozen=True, slots=True) +class RunningEdge: + edge: ProviderEdge + server: _EdgeHTTPServer + + def shutdown(self) -> None: + self.server.shutdown() + self.server.server_close() + + +def start_provider_edge( + backend: EdgeBackend, + *, + mounts: Mapping[str, str] = EDGE_MOUNTS, + bind_host: str = "127.0.0.1", + advertise_host: str | None = None, + forward_timeout: float = 60.0, +) -> RunningEdge: + """Boot an edge server on an OS-assigned port in a daemon thread. + ``advertise_host`` is what api_base URLs name (it differs from the bind + host when the proxy runs in a container and reaches the host machine via + a gateway address like host.docker.internal).""" + server: Final = _EdgeHTTPServer( + (bind_host, 0), backend=backend, mounts=mounts, forward_timeout=forward_timeout + ) + thread: Final = threading.Thread(target=server.serve_forever, name="e2e-provider-edge", daemon=True) + thread.start() + return RunningEdge( + edge=ProviderEdge(port=server.server_address[1], advertise_host=advertise_host or bind_host), + server=server, + ) + + +@functools.lru_cache(maxsize=8) +def _shared_recorder(root: Path) -> BundleRecorder: + prepared = prepare_bundle(root) + if isinstance(prepared, UnsafeBundleDir): + raise ValueError(f"E2E_FIXTURE_DIR {prepared.path} {prepared.reason}") + return prepared + + +@functools.lru_cache(maxsize=8) +def _shared_replay_source(root: Path) -> ReplaySource: + loaded = load_bundle(root) + if isinstance(loaded, UnreadableBundle): + raise ValueError(f"cannot replay from {root}: {loaded.reason}") + return ReplaySource(bundle=loaded) + + +@functools.lru_cache(maxsize=8) +def _shared_edge( + mode: Literal["record", "replay"], + bundle_dir: Path, + bind_host: str, + advertise_host: str, + forward_timeout: float, +) -> ProviderEdge: + backend: Final[EdgeBackend] = ( + RecordEdge(recorder=_shared_recorder(bundle_dir), lock=threading.Lock()) + if mode == "record" + else ReplayEdge(source=_shared_replay_source(bundle_dir)) + ) + return start_provider_edge( + backend, + mounts=EDGE_MOUNTS, + bind_host=bind_host, + advertise_host=advertise_host, + forward_timeout=forward_timeout, + ).edge + + +def replay_leftover_error(*, mode_raw: str, bundle_dir: Path, test_key: str) -> str | None: + """Teardown-time completeness check: in replay mode a passed test with + unconsumed recorded interactions must fail instead of passing against a + recording it no longer matches. Inert in every other mode.""" + if parse_fixture_mode(mode_raw) != "replay": + return None + return _shared_replay_source(bundle_dir).leftover_error(test_key) + + +def provider_edge_api_base( + mount: str, + *, + mode_raw: str, + bundle_dir: Path, + bind_host: str, + advertise_host: str, + forward_timeout: float = 60.0, +) -> str | None: + """The api_base a suite gives an edge-wired deployment: None in live mode + (the deployment keeps its real provider api_base) and the process-wide edge + server's mount URL in record and replay, booting the server on first use.""" + mode: Final = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode(value=value): + raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") + case "live": + return None + case "record" | "replay": + if mount not in EDGE_MOUNTS: + raise ValueError( + f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(EDGE_MOUNTS))}" + ) + return _shared_edge(mode, bundle_dir, bind_host, advertise_host, forward_timeout).api_base(mount) + case _: + assert_never(mode) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 3cae337a5ff..6cdd3354bf7 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -65,8 +65,6 @@ from models import ( ) from e2e_config import ( CONTROL_PLANE_BASE_URL, - FIXTURE_DIR, - FIXTURE_MODE_RAW, MASTER_KEY, POLL_INTERVAL, POLL_TIMEOUT, @@ -74,7 +72,6 @@ from e2e_config import ( REQUEST_TIMEOUT, settle_propagation, ) -from fixture_transport import select_transport from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] @@ -547,9 +544,9 @@ def build_proxy_client( pass all three together, since a caller that overrides only the data plane would leave management calls pointed at the env default. - E2E_FIXTURE_MODE wraps (record) or replaces (replay) the transport here, so - every client built from this seam records or replays without changing shape; - unset it stays the plain SplitTransport (see fixture_transport.py).""" + Test-to-proxy traffic always goes over the wire, in every E2E_FIXTURE_MODE: + record and replay scope to the proxy's provider-bound calls via the + provider edge (see provider_edge.py), never to this transport.""" split = SplitTransport( data=HttpTransport( base_url=base_url, @@ -563,12 +560,7 @@ def build_proxy_client( ), ) return ProxyClient( - transport=select_transport( - split, - mode_raw=FIXTURE_MODE_RAW, - bundle_dir=FIXTURE_DIR, - master_key=master_key, - ), + transport=split, poll_timeout=POLL_TIMEOUT, poll_interval=POLL_INTERVAL, ) diff --git a/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py b/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py new file mode 100644 index 00000000000..ced7c819d42 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py @@ -0,0 +1,50 @@ +"""The provider-edge demonstrator: one spend-tracking flow wired through the +record/replay edge (LIT-5745). + +This is the reference for wiring a suite to the edge: register a deployment +whose ``api_base`` comes from ``e2e_config.provider_edge_base``, then exercise +the proxy exactly as a live test would. In live mode the base is None and the +deployment talks to the real provider; in record mode it talks through the +local edge, which forwards to the provider and captures the exchange; in +replay mode the same test drives the REAL proxy and REAL database on the +recorded provider traffic alone, so key auth, routing, and the spend-log +write path are all still under test with zero provider calls. +""" + +import pytest + +from e2e_config import CHEAP_OPENAI_MODEL, provider_edge_base +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from spend_e2e_client import SpendClient, unique_marker, unwrap + +pytestmark = pytest.mark.e2e + + +@pytest.mark.covers("quota_management.spend_tracking.chat_completions.logs_cost") +def test_edge_wired_chat_writes_nonzero_spend_row( + client: SpendClient, resources: ResourceManager, scoped_key: str +) -> None: + base = provider_edge_base("openai") + model = f"e2e-edge-openai-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=f"openai/{CHEAP_OPENAI_MODEL}", + api_key="os.environ/OPENAI_API_KEY", + api_base=None if base is None else f"{base}/v1", + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + chat = unwrap( + client.chat(scoped_key, model, f"reply with one word {unique_marker()}", max_tokens=16) + ) + assert chat.id + + rows = client.poll_logs_for_key( + scoped_key, predicate=lambda rs: any((r.spend or 0) > 0 for r in rs) + ) + matching = [row for row in rows if row.request_id == chat.id] + assert matching, f"no SpendLogs row for request_id {chat.id}; saw {len(rows)} row(s)" + assert (matching[0].spend or 0) > 0, f"spend row for {chat.id} has zero spend" diff --git a/tests/e2e/test_fixture_bundle.py b/tests/e2e/test_fixture_bundle.py index fd4cca6451f..b49ab565e39 100644 --- a/tests/e2e/test_fixture_bundle.py +++ b/tests/e2e/test_fixture_bundle.py @@ -1,9 +1,9 @@ -"""Harness coverage for the on-disk fixture bundle format (LIT-5729). +"""Harness coverage for the on-disk fixture bundle format (LIT-5729/LIT-5745). No proxy and no ``e2e`` marker: these pin the bundle CONTRACT - the seven-day freshness gate that names the bundle's age, record mode's wipe safety (never delete a directory that is not a bundle), collision-free per-test slugs, and -lossless Result round-trips - so replay can never silently drift from what +grouped-in-order loading - so replay can never silently drift from what record wrote. """ @@ -12,18 +12,6 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone from pathlib import Path -import pytest -from pydantic import BaseModel - -from e2e_http import ( - NetworkError, - RateLimitedError, - Result, - Success, - UnauthorizedError, - UnknownApiError, - ValidationError, -) from fixture_bundle import ( BUNDLE_FORMAT_VERSION, MANIFEST_FILENAME, @@ -32,28 +20,22 @@ from fixture_bundle import ( FreshBundle, LoadedBundle, Manifest, + RecordedHttpResponse, RecordedRequest, - RecordedResult, StaleBundle, UnreadableBundle, UnsafeBundleDir, check_freshness, format_age, - from_result, interaction_filename, load_bundle, prepare_bundle, slug_for_test, - to_result, ) NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) -class Payload(BaseModel): - value: str - - def write_manifest( root: Path, recorded_at: datetime, *, format_version: int = BUNDLE_FORMAT_VERSION ) -> None: @@ -74,20 +56,8 @@ def plain_request(path: str) -> RecordedRequest: return RecordedRequest(method="post", path=path, headers={}) -class TestResultRoundTrip: - @pytest.mark.parametrize( - "result", - [ - Success(status_code=201, data=Payload(value="ok")), - NetworkError(message="connection refused"), - UnauthorizedError(), - RateLimitedError(retry_after_seconds=7, body="slow down"), - ValidationError(message="bad shape"), - UnknownApiError(status_code=502, body="upstream exploded"), - ], - ) - def test_every_result_kind_survives_disk_and_back(self, result: Result[Payload]) -> None: - assert to_result(from_result(result), Payload) == result +def plain_response() -> RecordedHttpResponse: + return RecordedHttpResponse(status_code=401, headers={}, body_b64="") class TestFreshness: @@ -144,7 +114,7 @@ class TestPrepareBundle: prepared(root).record( test_key="old.py::test_old", request=plain_request("/stale"), - response=RecordedResult(kind="unauthorized"), + response=plain_response(), ) assert any(entry.is_dir() for entry in root.iterdir()) prepared(root) @@ -193,7 +163,7 @@ class TestRecordAndLoad: recorder.record( test_key=key, request=plain_request(path), - response=RecordedResult(kind="unauthorized"), + response=plain_response(), ) loaded = load_bundle(root) assert isinstance(loaded, LoadedBundle) @@ -208,7 +178,7 @@ class TestRecordAndLoad: recorder.record( test_key=key, request=plain_request(f"/{key[-3:]}"), - response=RecordedResult(kind="unauthorized"), + response=plain_response(), ) loaded = load_bundle(root) assert isinstance(loaded, LoadedBundle) diff --git a/tests/e2e/test_fixture_mode.py b/tests/e2e/test_fixture_mode.py new file mode 100644 index 00000000000..109bb9e1b11 --- /dev/null +++ b/tests/e2e/test_fixture_mode.py @@ -0,0 +1,114 @@ +"""Harness coverage for fixture-mode selection and determinism (LIT-5729/LIT-5745). + +No proxy and no ``e2e`` marker. Pins the mode parser, the deterministic +per-test marker sequence a replay run must regenerate, the collection-time +gate (including the stale message that names the bundle's age), and the pytest +report header. The provider-edge record/replay behavior itself is pinned in +test_provider_edge.py. +""" + +from __future__ import annotations + +import hashlib +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from fixture_bundle import BUNDLE_FORMAT_VERSION, MANIFEST_FILENAME, Manifest +from fixture_mode import ( + InvalidFixtureMode, + current_test_key, + deterministic_marker, + fixture_mode_collection_error, + fixture_report_lines, + parse_fixture_mode, +) + +NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) + + +def write_manifest(root: Path, recorded_at: datetime) -> None: + root.mkdir(parents=True, exist_ok=True) + manifest = Manifest( + format_version=BUNDLE_FORMAT_VERSION, recorded_at=recorded_at, harness_version="abc1234" + ) + (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8") + + +class TestParseFixtureMode: + @pytest.mark.parametrize( + ("raw", "expected"), + [("live", "live"), ("record", "record"), ("replay", "replay"), ("", "live"), (" REPLAY ", "replay")], + ) + def test_known_values_normalize(self, raw: str, expected: str) -> None: + assert parse_fixture_mode(raw) == expected + + def test_unknown_value_is_invalid_with_the_original_spelling(self) -> None: + assert parse_fixture_mode("cached") == InvalidFixtureMode(value="cached") + + +class TestDeterministicMarker: + def test_sequence_is_a_pure_function_of_test_and_ordinal(self) -> None: + """A replay process must regenerate exactly the markers the record + process generated, so the Nth marker of a test is pinned to a pure + function of the node id and N.""" + key = current_test_key() + assert deterministic_marker() == hashlib.sha1(f"{key}#0".encode()).hexdigest()[:12] + assert deterministic_marker() == hashlib.sha1(f"{key}#1".encode()).hexdigest()[:12] + + +class TestCurrentTestKey: + def test_names_this_test_and_strips_the_phase(self) -> None: + key = current_test_key() + assert key.endswith("TestCurrentTestKey::test_names_this_test_and_strips_the_phase") + assert "(call)" not in key + + +class TestCollectionGate: + def test_invalid_mode_names_the_value_and_the_choices(self, tmp_path: Path) -> None: + assert ( + fixture_mode_collection_error("cached", tmp_path, now=NOW) + == "E2E_FIXTURE_MODE='cached' is not one of live, record, replay" + ) + + @pytest.mark.parametrize("mode_raw", ["live", "", "record"]) + def test_live_and_record_never_block_collection(self, mode_raw: str, tmp_path: Path) -> None: + assert fixture_mode_collection_error(mode_raw, tmp_path / "missing", now=NOW) is None + + def test_replay_with_no_bundle_says_how_to_record_one(self, tmp_path: Path) -> None: + reason = fixture_mode_collection_error("replay", tmp_path / "missing", now=NOW) + assert reason is not None + assert f"no {MANIFEST_FILENAME}" in reason + assert "E2E_FIXTURE_MODE=record" in reason + + def test_stale_replay_bundle_fails_naming_its_age(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=9, hours=5)) + reason = fixture_mode_collection_error("replay", root, now=NOW) + assert reason is not None + assert "age 9d5h exceeds the 7-day limit" in reason + assert "re-record with E2E_FIXTURE_MODE=record" in reason + + def test_fresh_replay_bundle_collects(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=2)) + assert fixture_mode_collection_error("replay", root, now=NOW) is None + + +class TestReportHeader: + def test_live_mode_prints_nothing(self, tmp_path: Path) -> None: + assert fixture_report_lines("live", tmp_path, now=NOW) == [] + assert fixture_report_lines("", tmp_path, now=NOW) == [] + + def test_record_and_replay_name_the_bundle(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorded_at = NOW - timedelta(days=1) + write_manifest(root, recorded_at) + assert fixture_report_lines("record", root, now=NOW) == [ + f"e2e fixture mode: record -> {root}" + ] + replay_lines = fixture_report_lines("replay", root, now=NOW) + assert len(replay_lines) == 1 + assert "replay" in replay_lines[0] + assert recorded_at.isoformat() in replay_lines[0] diff --git a/tests/e2e/test_fixture_transport.py b/tests/e2e/test_fixture_transport.py deleted file mode 100644 index e61088d841c..00000000000 --- a/tests/e2e/test_fixture_transport.py +++ /dev/null @@ -1,676 +0,0 @@ -"""Harness coverage for the record/replay transports (LIT-5729). - -No proxy and no ``e2e`` marker. A fake in-memory ``Transport`` stands in for -the live one (dependency injection, no monkeypatching): recording must pass -every value through unchanged while writing one redacted interaction file per -call, and replay must serve identical values from the bundle alone - the -fake's call log proves nothing reaches the inner transport - failing hard -(``ReplayMiss``) on any content drift, printing the computed canonical key and -the closest recorded key (LIT-5741; the pure canonicalizer is pinned in -test_fixture_canonical.py). The collection-time gate and report header are -pinned here too, including the stale message that names the bundle's age. -""" - -from __future__ import annotations - -import hashlib -import sys -import threading -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field -from datetime import datetime, timedelta, timezone -from pathlib import Path -from uuid import uuid4 - -import pytest -from pydantic import BaseModel - -from e2e_http import ( - AuthHeaders, - BinaryStream, - ProbeResult, - Result, - StreamingResponse, - Success, -) -from fixture_bundle import ( - BUNDLE_FORMAT_VERSION, - MANIFEST_FILENAME, - BundleRecorder, - Interaction, - LoadedBundle, - Manifest, - RecordedResult, - load_bundle, - prepare_bundle, - slug_for_test, -) -from fixture_canonical import canonicalize -from fixture_transport import ( - InvalidFixtureMode, - RecordingTransport, - ReplayMiss, - ReplaySource, - ReplayTransport, - current_test_key, - deterministic_marker, - fixture_mode_collection_error, - fixture_report_lines, - parse_fixture_mode, - recorded_request, - replay_leftover_error, - select_transport, -) -from transport import Transport - -NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) - - -class Payload(BaseModel): - value: str - - -class Body(BaseModel): - prompt: str - - -class Query(BaseModel): - q: str - - -class DeployParams(BaseModel): - model: str - api_key: str | None = None - aws_secret_access_key: str | None = None - - -class DeployBody(BaseModel): - model_name: str - litellm_params: DeployParams - - -STREAMING = StreamingResponse( - status_code=200, - body="", - content_type="text/event-stream", - chunks=2, - stream_events=["one", "two"], - stream_done=True, -) -BINARY = BinaryStream(status_code=200, content_type="audio/mpeg", chunk_count=3, total_bytes=42) -PROBE = ProbeResult(status_code=200, body="alive") - - -@dataclass -class FakeTransport: - calls: list[str] = field(default_factory=list) - - def bearer(self, key: str) -> AuthHeaders: - return AuthHeaders(authorization=f"Bearer {key}") - - @property - def master(self) -> AuthHeaders: - return self.bearer("sk-fake-master") - - def _success[R: BaseModel](self, response_type: type[R]) -> Result[R]: - return Success(status_code=200, data=response_type.model_validate({"value": "live"})) - - def post[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - self.calls.append(f"post {path}") - return self._success(response_type) - - def get[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - params: BaseModel, - response_type: type[R], - timeout: float | None = None, - ) -> Result[R]: - self.calls.append(f"get {path}") - return self._success(response_type) - - def delete[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - response_type: type[R], - params: BaseModel | None = None, - ) -> Result[R]: - self.calls.append(f"delete {path}") - return self._success(response_type) - - def patch[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - self.calls.append(f"patch {path}") - return self._success(response_type) - - def put[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - self.calls.append(f"put {path}") - return self._success(response_type) - - def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: - self.calls.append(f"stream {path}") - return STREAMING - - def stream_binary( - self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 - ) -> BinaryStream: - self.calls.append(f"stream_binary {path}") - return BINARY - - def send( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - params: BaseModel | None = None, - stream: bool = False, - ) -> StreamingResponse: - self.calls.append(f"send {path}") - return STREAMING - - def probe(self, path: str, *, params: BaseModel) -> ProbeResult: - self.calls.append(f"probe {path}") - return PROBE - - def upload[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - form: BaseModel, - filename: str, - content: bytes, - file_content_type: str = "application/jsonl", - file_field: str = "file", - params: BaseModel | None = None, - response_type: type[R], - ) -> Result[R]: - self.calls.append(f"upload {path}") - return self._success(response_type) - - def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: - self.calls.append(f"download {path}") - return STREAMING - - -def make_recorder(root: Path) -> BundleRecorder: - recorder = prepare_bundle(root) - assert isinstance(recorder, BundleRecorder) - return recorder - - -def replay_source(root: Path) -> ReplaySource: - loaded = load_bundle(root) - assert isinstance(loaded, LoadedBundle) - return ReplaySource(bundle=loaded) - - -def this_tests_files(root: Path) -> list[Path]: - slug_dir = root / slug_for_test(current_test_key()) - return sorted(slug_dir.glob("*.json")) if slug_dir.is_dir() else [] - - -def write_manifest(root: Path, recorded_at: datetime) -> None: - root.mkdir(parents=True, exist_ok=True) - manifest = Manifest( - format_version=BUNDLE_FORMAT_VERSION, recorded_at=recorded_at, harness_version="abc1234" - ) - (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8") - - -class TestParseFixtureMode: - @pytest.mark.parametrize( - ("raw", "expected"), - [("live", "live"), ("record", "record"), ("replay", "replay"), ("", "live"), (" REPLAY ", "replay")], - ) - def test_known_values_normalize(self, raw: str, expected: str) -> None: - assert parse_fixture_mode(raw) == expected - - def test_unknown_value_is_invalid_with_the_original_spelling(self) -> None: - assert parse_fixture_mode("cached") == InvalidFixtureMode(value="cached") - - -class TestDeterministicMarker: - def test_sequence_is_a_pure_function_of_test_and_ordinal(self) -> None: - """A replay process must regenerate exactly the markers the record - process generated, so the Nth marker of a test is pinned to a pure - function of the node id and N.""" - key = current_test_key() - assert deterministic_marker() == hashlib.sha1(f"{key}#0".encode()).hexdigest()[:12] - assert deterministic_marker() == hashlib.sha1(f"{key}#1".encode()).hexdigest()[:12] - - -class TestCurrentTestKey: - def test_names_this_test_and_strips_the_phase(self) -> None: - key = current_test_key() - assert key.endswith("TestCurrentTestKey::test_names_this_test_and_strips_the_phase") - assert "(call)" not in key - - -class TestRecordingTransport: - def test_passes_the_result_through_and_writes_one_file_per_call(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - result = recording.post( - "/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload - ) - assert result == Success(status_code=200, data=Payload(value="live")) - assert fake.calls == ["post /model/new"] - files = this_tests_files(root) - assert [file.name for file in files] == ["0000-post-model-new.json"] - interaction = Interaction.model_validate_json(files[0].read_text(encoding="utf-8")) - assert interaction.request.method == "post" - assert interaction.request.path == "/model/new" - - def test_redacts_auth_header_values_in_the_recorded_request(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - headers = AuthHeaders.model_validate( - {"authorization": "Bearer sk-secret", "x-litellm-api-key": "sk-other"} - ) - recording.post("/key/generate", headers=headers, json=Body(prompt="x"), response_type=Payload) - interaction = Interaction.model_validate_json( - this_tests_files(root)[0].read_text(encoding="utf-8") - ) - assert interaction.request.headers == { - "authorization": "", - "x-litellm-api-key": "", - } - assert "sk-secret" not in this_tests_files(root)[0].read_text(encoding="utf-8") - - def test_redacts_credential_body_fields_in_the_recorded_request(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post( - "/model/new", - headers=fake.master, - json=DeployBody( - model_name="m", - litellm_params=DeployParams(model="openai/gpt", api_key="sk-live-provider-secret-123456"), - ), - response_type=Payload, - ) - raw = this_tests_files(root)[0].read_text(encoding="utf-8") - interaction = Interaction.model_validate_json(raw) - assert "sk-live-provider-secret-123456" not in raw - assert isinstance(interaction.request.body, dict) - params = interaction.request.body["litellm_params"] - assert isinstance(params, dict) - assert params["api_key"] == "" - assert params["aws_secret_access_key"] is None - - def test_upload_records_a_content_digest_not_the_bytes(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.upload( - "/v1/files", - headers=fake.master, - form=Query(q="batch"), - filename="batch.jsonl", - content=b'{"custom_id": "1"}', - response_type=Payload, - ) - interaction = Interaction.model_validate_json( - this_tests_files(root)[0].read_text(encoding="utf-8") - ) - assert interaction.request.file_name == "batch.jsonl" - assert interaction.request.file_bytes == len(b'{"custom_id": "1"}') - assert interaction.request.file_sha256 is not None - assert "custom_id" not in interaction.request.model_dump_json() - - -class TestReplayTransport: - def test_serves_recorded_values_without_touching_the_inner_transport( - self, tmp_path: Path - ) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recorded_post = recording.post( - "/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload - ) - recorded_get = recording.get( - "/v1/models", headers=fake.master, params=Query(q="all"), response_type=Payload - ) - recorded_stream = recording.stream( - "/chat/completions", headers=fake.master, json=Body(prompt="hi") - ) - recorded_probe = recording.probe("/health/liveliness", params=Query(q="1")) - recorded_binary = recording.stream_binary( - "/v1/audio/speech", headers=fake.master, json=Body(prompt="say") - ) - calls_after_record = list(fake.calls) - - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - assert ( - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - == recorded_post - ) - assert ( - replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) - == recorded_get - ) - assert ( - replay.stream("/chat/completions", headers=replay.master, json=Body(prompt="hi")) - == recorded_stream - ) - assert replay.probe("/health/liveliness", params=Query(q="1")) == recorded_probe - assert ( - replay.stream_binary("/v1/audio/speech", headers=replay.master, json=Body(prompt="say")) - == recorded_binary - ) - assert fake.calls == calls_after_record - - def test_miss_names_the_computed_key_and_the_closest_recorded_key(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - with pytest.raises(ReplayMiss) as excinfo: - replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) - message = str(excinfo.value) - assert "no recorded interaction matches key get /v1/models #" in message - assert "closest recorded key is post /model/new #" in message - assert "0000-post-model-new.json" in message - assert "re-record with E2E_FIXTURE_MODE=record" in message - - def test_content_drift_on_the_same_route_misses_with_no_live_call(self, tmp_path: Path) -> None: - """The naive verb+path match replayed a stale response for a request - whose content had changed, silently passing; a content key must miss, - print both canonical forms' diff, and never reach the inner transport.""" - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - calls_after_record = list(fake.calls) - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - with pytest.raises(ReplayMiss) as excinfo: - replay.post("/model/new", headers=replay.master, json=Body(prompt="y"), response_type=Payload) - message = str(excinfo.value) - assert "no recorded interaction matches key post /model/new #" in message - assert "closest recorded key is post /model/new #" in message - assert '- "prompt": "x"' in message - assert '+ "prompt": "y"' in message - assert fake.calls == calls_after_record - - def test_exhausted_key_names_the_key(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - with pytest.raises( - ReplayMiss, match=r"every recorded interaction for key post /model/new #\w{16} is already consumed" - ): - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - - def test_replays_out_of_recorded_order_across_distinct_keys(self, tmp_path: Path) -> None: - """Concurrent tests interleave independent calls nondeterministically - (e.g. a burst of parallel chat calls), so replay matches by content, - never by recorded position.""" - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - recording.post("/key/generate", headers=fake.master, json=Body(prompt="k"), response_type=Payload) - source = replay_source(root) - replay: Transport = ReplayTransport(source=source, master_key="sk-1234") - replay.post("/key/generate", headers=replay.master, json=Body(prompt="k"), response_type=Payload) - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - assert source.leftover_error(current_test_key()) is None - - def test_identical_requests_replay_their_responses_in_recorded_order(self, tmp_path: Path) -> None: - """A poll loop makes the same request repeatedly and asserts on the - progression, so duplicates under one key stay FIFO.""" - root = tmp_path / "bundle" - recorder = make_recorder(root) - recorder.record( - test_key=current_test_key(), - request=recorded_request( - "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") - ), - response=RecordedResult(kind="success", status_code=200, data={"value": "first"}), - ) - recorder.record( - test_key=current_test_key(), - request=recorded_request( - "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") - ), - response=RecordedResult(kind="success", status_code=200, data={"value": "second"}), - ) - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - first = replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) - second = replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) - assert first == Success(status_code=200, data=Payload(value="first")) - assert second == Success(status_code=200, data=Payload(value="second")) - - def test_concurrent_replays_of_one_key_serve_each_recording_exactly_once(self, tmp_path: Path) -> None: - """A burst of parallel identical calls consumes one shared pool: no - response duplicated, none forgotten, nothing left over at teardown. - The tiny switch interval forces thread preemption inside pool setup - and consumption, so a non-atomic pool build or pop fails this test.""" - root = tmp_path / "bundle" - recorder = make_recorder(root) - for ordinal in range(32): - recorder.record( - test_key=current_test_key(), - request=recorded_request( - "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") - ), - response=RecordedResult(kind="success", status_code=200, data={"value": f"v{ordinal:02d}"}), - ) - source = replay_source(root) - replay: Transport = ReplayTransport(source=source, master_key="sk-1234") - barrier = threading.Barrier(8) - - def consume_one() -> str: - result = replay.get( - "/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload - ) - assert isinstance(result, Success) - return result.data.value - - def consume(_: int) -> tuple[str, ...]: - barrier.wait() - return tuple(consume_one() for _call in range(4)) - - previous_interval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) - try: - with ThreadPoolExecutor(max_workers=8) as executor: - served = sorted(value for values in executor.map(consume, range(8)) for value in values) - finally: - sys.setswitchinterval(previous_interval) - assert served == [f"v{ordinal:02d}" for ordinal in range(32)] - assert source.leftover_error(current_test_key()) is None - - -class TestRecordedKeySets: - def test_two_separate_recordings_of_one_flow_produce_identical_key_sets( - self, tmp_path: Path - ) -> None: - """Everything a run randomizes (markers, virtual keys, dates) must - canonicalize out, so separately recorded runs of the same suite agree - on every match key and a bundle recorded elsewhere replays here.""" - - def record_flow(root: Path, run_date: str) -> list[str]: - fake = FakeTransport() - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - marker = deterministic_marker() - recording.post( - "/model/new", - headers=fake.master, - json=DeployBody( - model_name=f"e2e-chat-{marker}", - litellm_params=DeployParams(model="openai/gpt", api_key=f"sk-live-{uuid4().hex}"), - ), - response_type=Payload, - ) - recording.post( - "/chat/completions", - headers=recording.bearer(f"sk-{uuid4().hex}"), - json=Body(prompt=f"Reply with the single word ok. {marker}"), - response_type=Payload, - ) - recording.get( - "/spend/logs", headers=fake.master, params=Query(q=run_date), response_type=Payload - ) - loaded = load_bundle(root) - assert isinstance(loaded, LoadedBundle) - return sorted( - canonicalize(interaction.request).key - for interactions in loaded.interactions.values() - for interaction in interactions - ) - - first_keys = record_flow(tmp_path / "one", "2026-08-18") - second_keys = record_flow(tmp_path / "two", "2026-08-19") - assert first_keys == second_keys - assert len(first_keys) == 3 - - -class TestReplayLeftover: - def test_fully_consumed_recording_leaves_nothing(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - source = replay_source(root) - replay: Transport = ReplayTransport(source=source, master_key="sk-1234") - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - assert source.leftover_error(current_test_key()) is None - - def test_unconsumed_trailing_interactions_name_the_next_call(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - recording.probe("/health/liveliness", params=Query(q="1")) - source = replay_source(root) - replay: Transport = ReplayTransport(source=source, master_key="sk-1234") - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - error = source.leftover_error(current_test_key()) - assert error is not None - assert "1 of 2 recorded interactions never consumed" in error - assert "e.g. probe /health/liveliness #" in error - assert "re-record with E2E_FIXTURE_MODE=record" in error - - def test_test_without_recordings_has_no_leftover(self, tmp_path: Path) -> None: - root = tmp_path / "bundle" - make_recorder(root) - assert replay_source(root).leftover_error("suite.py::test_never_recorded") is None - - def test_inert_outside_replay_mode(self, tmp_path: Path) -> None: - missing = tmp_path / "missing" - assert replay_leftover_error(mode_raw="", bundle_dir=missing, test_key="k") is None - assert replay_leftover_error(mode_raw="record", bundle_dir=missing, test_key="k") is None - - def test_replay_mode_reads_the_shared_bundle(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - error = replay_leftover_error(mode_raw="replay", bundle_dir=root, test_key=current_test_key()) - assert error is not None - assert "1 of 1 recorded interactions never consumed" in error - - -class TestSelectTransport: - def test_live_returns_the_live_transport_untouched(self, tmp_path: Path) -> None: - fake = FakeTransport() - for mode_raw in ("live", ""): - assert ( - select_transport(fake, mode_raw=mode_raw, bundle_dir=tmp_path / "b", master_key="sk") - is fake - ) - - def test_record_wraps_live_and_starts_a_fresh_bundle(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - write_manifest(root, NOW - timedelta(days=30)) - (root / "old-test-slug").mkdir() - (root / "old-test-slug" / "0000-post-old.json").write_text("{}", encoding="utf-8") - selected = select_transport(fake, mode_raw="record", bundle_dir=root, master_key="sk") - assert isinstance(selected, RecordingTransport) - assert selected.inner is fake - assert {entry.name for entry in root.iterdir()} == {MANIFEST_FILENAME} - - def test_replay_builds_a_transport_from_the_bundle_alone(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - make_recorder(root) - selected = select_transport(fake, mode_raw="replay", bundle_dir=root, master_key="sk-master") - assert isinstance(selected, ReplayTransport) - assert selected.master == AuthHeaders(authorization="Bearer sk-master") - - def test_invalid_mode_raises_naming_the_value(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="cached"): - select_transport( - FakeTransport(), mode_raw="cached", bundle_dir=tmp_path / "b", master_key="sk" - ) - - -class TestCollectionGate: - def test_invalid_mode_names_the_value_and_the_choices(self, tmp_path: Path) -> None: - assert ( - fixture_mode_collection_error("cached", tmp_path, now=NOW) - == "E2E_FIXTURE_MODE='cached' is not one of live, record, replay" - ) - - @pytest.mark.parametrize("mode_raw", ["live", "", "record"]) - def test_live_and_record_never_block_collection(self, mode_raw: str, tmp_path: Path) -> None: - assert fixture_mode_collection_error(mode_raw, tmp_path / "missing", now=NOW) is None - - def test_replay_with_no_bundle_says_how_to_record_one(self, tmp_path: Path) -> None: - reason = fixture_mode_collection_error("replay", tmp_path / "missing", now=NOW) - assert reason is not None - assert f"no {MANIFEST_FILENAME}" in reason - assert "E2E_FIXTURE_MODE=record" in reason - - def test_stale_replay_bundle_fails_naming_its_age(self, tmp_path: Path) -> None: - root = tmp_path / "bundle" - write_manifest(root, NOW - timedelta(days=9, hours=5)) - reason = fixture_mode_collection_error("replay", root, now=NOW) - assert reason is not None - assert "age 9d5h exceeds the 7-day limit" in reason - assert "re-record with E2E_FIXTURE_MODE=record" in reason - - def test_fresh_replay_bundle_collects(self, tmp_path: Path) -> None: - root = tmp_path / "bundle" - write_manifest(root, NOW - timedelta(days=2)) - assert fixture_mode_collection_error("replay", root, now=NOW) is None - - -class TestReportHeader: - def test_live_mode_prints_nothing(self, tmp_path: Path) -> None: - assert fixture_report_lines("live", tmp_path, now=NOW) == [] - assert fixture_report_lines("", tmp_path, now=NOW) == [] - - def test_record_and_replay_name_the_bundle(self, tmp_path: Path) -> None: - root = tmp_path / "bundle" - recorded_at = NOW - timedelta(days=1) - write_manifest(root, recorded_at) - assert fixture_report_lines("record", root, now=NOW) == [ - f"e2e fixture mode: record -> {root}" - ] - replay_lines = fixture_report_lines("replay", root, now=NOW) - assert len(replay_lines) == 1 - assert "replay" in replay_lines[0] - assert recorded_at.isoformat() in replay_lines[0] diff --git a/tests/e2e/test_provider_edge.py b/tests/e2e/test_provider_edge.py new file mode 100644 index 00000000000..492eee57aaf --- /dev/null +++ b/tests/e2e/test_provider_edge.py @@ -0,0 +1,492 @@ +"""Harness coverage for the provider-edge record/replay server (LIT-5745). + +No proxy and no ``e2e`` marker. A stdlib http.server stands in for the +provider (dependency injection via the mounts mapping, no monkeypatching): +record mode must forward each edge call to it verbatim, persist one +interaction file, and serve the proxy the same filtered response replay will +serve later; replay mode must serve byte-identical responses from the bundle +alone, with the fake provider's hit log proving nothing leaves the process, +and answer any drifted call with HTTP ``REPLAY_MISS_STATUS`` naming the +computed and closest recorded canonical keys (LIT-5741; the pure canonicalizer +is pinned in test_fixture_canonical.py). Requests are made through +``e2e_http.forward`` so the whole HTTP surface of the edge is exercised; the +pure ``handle_edge_request`` core is pinned socket-free alongside. +""" + +from __future__ import annotations + +import base64 +import json +import threading +from collections.abc import Generator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from pydantic import TypeAdapter + +from e2e_http import RawResponse, forward +from fixture_bundle import ( + BundleRecorder, + Interaction, + LoadedBundle, + RecordedHttpResponse, + RecordedRequest, + load_bundle, + prepare_bundle, + slug_for_test, +) +from fixture_mode import current_test_key +from provider_edge import ( + REPLAY_MISS_STATUS, + EdgeBackend, + ProviderEdge, + RecordEdge, + ReplayEdge, + ReplaySource, + handle_edge_request, + provider_edge_api_base, + replay_leftover_error, + start_provider_edge, +) + +CHAT_PATH = "/openai/v1/chat/completions" +REPLAY_MOUNTS = {"openai": "https://replay.invalid"} +JSON_OBJECT = TypeAdapter(dict[str, object]) + + +def json_object(body: bytes) -> dict[str, object]: + return JSON_OBJECT.validate_json(body) + + +class _FakeProvider(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, bind: tuple[str, int]) -> None: + super().__init__(bind, _FakeProviderHandler) + self.hits: list[str] = [] + + +class _FakeProviderHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + self._respond() + + def do_GET(self) -> None: + self._respond() + + def _respond(self) -> None: + provider = self.server + assert isinstance(provider, _FakeProvider) + length = int(self.headers.get("content-length") or "0") + body = self.rfile.read(length) if length else b"" + provider.hits.append(f"{self.command} {self.path}") + payload = json.dumps( + {"echo": body.decode("utf-8"), "path": self.path, "hit": len(provider.hits)} + ).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(payload))) + self.send_header("x-upstream", "fake") + self.send_header("set-cookie", "session=fake-cookie") + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format: str, *args: object) -> None: + """Silence the per-request stderr line BaseHTTPRequestHandler emits.""" + + +@contextmanager +def fake_provider() -> Generator[_FakeProvider]: + server = _FakeProvider(("127.0.0.1", 0)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + + +def provider_url(server: _FakeProvider) -> str: + return f"http://127.0.0.1:{server.server_address[1]}" + + +@contextmanager +def running_edge(backend: EdgeBackend, mounts: Mapping[str, str]) -> Generator[ProviderEdge]: + running = start_provider_edge(backend, mounts=mounts, bind_host="127.0.0.1") + try: + yield running.edge + finally: + running.shutdown() + + +def record_backend(root: Path) -> RecordEdge: + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + return RecordEdge(recorder=recorder, lock=threading.Lock()) + + +def replay_source(root: Path) -> ReplaySource: + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + return ReplaySource(bundle=loaded) + + +def call_edge( + edge: ProviderEdge, + method: str, + path: str, + *, + body: bytes | None = None, + headers: dict[str, str] | None = None, +) -> RawResponse: + outcome = forward( + method, + f"http://{edge.advertise_host}:{edge.port}{path}", + headers=headers or {}, + body=body, + timeout=10.0, + ) + assert isinstance(outcome, RawResponse) + return outcome + + +def this_tests_files(root: Path) -> list[Path]: + slug_dir = root / slug_for_test(current_test_key()) + return sorted(slug_dir.glob("*.json")) if slug_dir.is_dir() else [] + + +def chat_body(prompt: str) -> bytes: + return json.dumps({"model": "gpt", "messages": [{"role": "user", "content": prompt}]}).encode() + + +class TestRecordMode: + def test_forwards_to_the_provider_and_writes_one_interaction_file(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + reply = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert provider.hits == ["POST /v1/chat/completions"] + assert reply.status_code == 200 + served = json_object(reply.body) + assert served["echo"] == chat_body("hi").decode() + files = this_tests_files(root) + assert [file.name for file in files] == ["0000-post-openai-v1-chat-completions.json"] + interaction = Interaction.model_validate_json(files[0].read_text(encoding="utf-8")) + assert interaction.request.method == "post" + assert interaction.request.path == CHAT_PATH + assert interaction.request.body == json_object(chat_body("hi")) + assert interaction.response.status_code == 200 + + def test_never_stores_headers_so_credentials_never_touch_disk(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge( + edge, + "POST", + CHAT_PATH, + body=chat_body("hi"), + headers={"authorization": "Bearer sk-live-provider-secret-abc123"}, + ) + raw = this_tests_files(root)[0].read_text(encoding="utf-8") + assert "sk-live-provider-secret-abc123" not in raw + interaction = Interaction.model_validate_json(raw) + assert interaction.request.headers == {} + + def test_strips_volatile_response_headers_and_serves_the_filtered_copy(self, tmp_path: Path) -> None: + """What record serves the proxy must equal what replay will serve later + (record/replay parity), so the filtered stored copy is served in both.""" + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + reply = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert reply.headers.get("x-upstream") == "fake" + assert "set-cookie" not in reply.headers + interaction = Interaction.model_validate_json( + this_tests_files(root)[0].read_text(encoding="utf-8") + ) + assert interaction.response.headers.get("x-upstream") == "fake" + assert "set-cookie" not in interaction.response.headers + assert "content-length" not in interaction.response.headers + + def test_unreachable_provider_records_and_serves_a_502(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with running_edge(record_backend(root), {"openai": "http://127.0.0.1:9"}) as edge: + reply = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert reply.status_code == 502 + assert b"could not reach the provider" in reply.body + interaction = Interaction.model_validate_json( + this_tests_files(root)[0].read_text(encoding="utf-8") + ) + assert interaction.response.status_code == 502 + + +class TestReplayMode: + def test_serves_recorded_bytes_with_zero_provider_hits(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + recorded = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + hits_after_record = list(provider.hits) + with running_edge( + ReplayEdge(source=replay_source(root)), {"openai": provider_url(provider)} + ) as edge: + replayed = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert provider.hits == hits_after_record + assert replayed.status_code == recorded.status_code + assert replayed.body == recorded.body + assert replayed.headers.get("x-upstream") == "fake" + + def test_request_identity_ignores_auth_headers(self, tmp_path: Path) -> None: + """The proxy sends different bearer tokens across runs (fresh virtual + keys, rotated provider keys), so headers are no part of the match.""" + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge( + edge, "POST", CHAT_PATH, body=chat_body("hi"), + headers={"authorization": "Bearer sk-first-run"}, + ) + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + replayed = call_edge( + edge, "POST", CHAT_PATH, body=chat_body("hi"), + headers={"authorization": "Bearer sk-second-run"}, + ) + assert replayed.status_code == 200 + + def test_content_drift_returns_the_miss_status_naming_both_keys(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("x")) + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + missed = call_edge(edge, "POST", CHAT_PATH, body=chat_body("y")) + assert missed.status_code == REPLAY_MISS_STATUS + message = missed.body.decode() + assert f"no recorded interaction matches key post {CHAT_PATH} #" in message + assert f"closest recorded key is post {CHAT_PATH} #" in message + assert '"content": "x"' in message + assert '"content": "y"' in message + assert "re-record with E2E_FIXTURE_MODE=record" in message + + def test_query_params_are_part_of_the_identity(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "GET", "/openai/v1/models?purpose=batch") + assert provider.hits == ["GET /v1/models?purpose=batch"] + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + missed = call_edge(edge, "GET", "/openai/v1/models?purpose=other") + matched = call_edge(edge, "GET", "/openai/v1/models?purpose=batch") + assert missed.status_code == REPLAY_MISS_STATUS + assert matched.status_code == 200 + + def test_identical_requests_replay_their_responses_in_recorded_order(self, tmp_path: Path) -> None: + """A poll or retry loop repeats the same request and the proxy asserts + on the progression, so duplicates under one key stay FIFO.""" + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + first = json_object(call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")).body) + second = json_object(call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")).body) + assert first["hit"] == 1 + assert second["hit"] == 2 + + def test_exhausted_key_returns_the_miss_status(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + exhausted = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert exhausted.status_code == REPLAY_MISS_STATUS + assert b"already consumed" in exhausted.body + + def test_non_json_bodies_match_by_canonical_digest_without_storing_them(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + opaque = b"custom_id one\ncustom_id two\n" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", "/openai/v1/files", body=opaque) + raw = this_tests_files(root)[0].read_text(encoding="utf-8") + interaction = Interaction.model_validate_json(raw) + assert interaction.request.body is None + assert interaction.request.file_sha256 is not None + assert interaction.request.file_bytes == len(opaque) + assert "custom_id" not in interaction.request.model_dump_json() + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + replayed = call_edge(edge, "POST", "/openai/v1/files", body=opaque) + assert replayed.status_code == 200 + + +class TestReplayLeftover: + def test_partially_consumed_recording_names_the_leftover(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + call_edge(edge, "GET", "/openai/v1/models") + source = replay_source(root) + with running_edge(ReplayEdge(source=source), REPLAY_MOUNTS) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + error = source.leftover_error(current_test_key()) + assert error is not None + assert "1 of 2 recorded interactions never consumed" in error + assert "e.g. get /openai/v1/models #" in error + assert "re-record with E2E_FIXTURE_MODE=record" in error + + def test_fully_consumed_recording_leaves_nothing(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + source = replay_source(root) + with running_edge(ReplayEdge(source=source), REPLAY_MOUNTS) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert source.leftover_error(current_test_key()) is None + + def test_test_without_recordings_has_no_leftover(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + assert isinstance(prepare_bundle(root), BundleRecorder) + assert replay_source(root).leftover_error("suite.py::test_never_recorded") is None + + def test_inert_outside_replay_mode(self, tmp_path: Path) -> None: + missing = tmp_path / "missing" + assert replay_leftover_error(mode_raw="", bundle_dir=missing, test_key="k") is None + assert replay_leftover_error(mode_raw="record", bundle_dir=missing, test_key="k") is None + + +class TestConcurrentReplay: + def test_parallel_identical_calls_serve_each_recording_exactly_once(self, tmp_path: Path) -> None: + """The edge server handles requests on concurrent threads and a burst + of parallel identical calls consumes one shared pool: no response + duplicated, none forgotten, nothing left over at teardown.""" + root = tmp_path / "bundle" + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + for ordinal in range(32): + recorder.record( + test_key=current_test_key(), + request=RecordedRequest(method="post", path=CHAT_PATH, headers={}, body={"n": "same"}), + response=RecordedHttpResponse( + status_code=200, + headers={"content-type": "application/json"}, + body_b64=base64.b64encode(json.dumps({"value": f"v{ordinal:02d}"}).encode()).decode(), + ), + ) + source = replay_source(root) + body = json.dumps({"n": "same"}).encode() + barrier = threading.Barrier(8) + with running_edge(ReplayEdge(source=source), REPLAY_MOUNTS) as edge: + + def consume(_: int) -> tuple[str, ...]: + barrier.wait() + return tuple( + str(json_object(call_edge(edge, "POST", CHAT_PATH, body=body).body)["value"]) + for _call in range(4) + ) + + with ThreadPoolExecutor(max_workers=8) as executor: + served = sorted(value for values in executor.map(consume, range(8)) for value in values) + assert served == [f"v{ordinal:02d}" for ordinal in range(32)] + assert source.leftover_error(current_test_key()) is None + + +class TestHandleEdgeRequestPure: + def test_unknown_mount_404s_naming_the_known_mounts(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + assert isinstance(prepare_bundle(root), BundleRecorder) + reply = handle_edge_request( + ReplayEdge(source=replay_source(root)), + {"openai": "https://api.openai.com", "anthropic": "https://api.anthropic.com"}, + "POST", + "/bedrock/model/invoke", + {}, + b"{}", + timeout=1.0, + ) + assert reply.status_code == 404 + assert b"unknown provider mount 'bedrock'" in reply.body + assert b"anthropic, openai" in reply.body + + def test_replay_serves_a_directly_recorded_interaction(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + recorder.record( + test_key=current_test_key(), + request=RecordedRequest(method="post", path=CHAT_PATH, headers={}, body={"prompt": "x"}), + response=RecordedHttpResponse( + status_code=201, headers={"x-upstream": "fake"}, body_b64=base64.b64encode(b"ok").decode() + ), + ) + reply = handle_edge_request( + ReplayEdge(source=replay_source(root)), + {"openai": "https://api.openai.com"}, + "POST", + CHAT_PATH, + {"authorization": "Bearer sk-anything"}, + json.dumps({"prompt": "x"}).encode(), + timeout=1.0, + ) + assert reply.status_code == 201 + assert reply.body == b"ok" + assert reply.headers == {"x-upstream": "fake"} + + +class TestApiBaseSeam: + def test_live_mode_returns_none(self, tmp_path: Path) -> None: + for mode_raw in ("live", ""): + assert ( + provider_edge_api_base( + "openai", + mode_raw=mode_raw, + bundle_dir=tmp_path / "bundle", + bind_host="127.0.0.1", + advertise_host="127.0.0.1", + ) + is None + ) + + def test_invalid_mode_raises_naming_the_value(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="cached"): + provider_edge_api_base( + "openai", + mode_raw="cached", + bundle_dir=tmp_path / "bundle", + bind_host="127.0.0.1", + advertise_host="127.0.0.1", + ) + + def test_unknown_mount_raises_naming_the_known_mounts(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="unknown provider mount 'bedrock'"): + provider_edge_api_base( + "bedrock", + mode_raw="record", + bundle_dir=tmp_path / "bundle", + bind_host="127.0.0.1", + advertise_host="127.0.0.1", + ) + + def test_record_mode_boots_one_shared_edge_and_prepares_the_bundle(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + first = provider_edge_api_base( + "openai", mode_raw="record", bundle_dir=root, bind_host="127.0.0.1", advertise_host="127.0.0.1" + ) + second = provider_edge_api_base( + "anthropic", mode_raw="record", bundle_dir=root, bind_host="127.0.0.1", advertise_host="127.0.0.1" + ) + assert first is not None and second is not None + assert first.endswith("/openai") + assert second.endswith("/anthropic") + assert first.rsplit("/", 1)[0] == second.rsplit("/", 1)[0] + assert (root / "manifest.json").is_file() From 0de829d3e44094a808e9c1166d12c4d86b29c6b0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:57:35 -0700 Subject: [PATCH 03/53] feat(cli): store the lite login credential in the OS keychain lite login used to write the minted cli-session key in cleartext to ~/.litellm/token.json. The secret material (key plus any JWT) now goes to the OS keychain through the optional keyring package, with the 0600 file kept for non-secret metadata and as the fallback on headless boxes. Legacy plaintext files keep authenticating and are migrated into the keychain, then scrubbed, on first read. A secret still on disk always outranks the keychain entry, so a failed keychain write can never resurrect a stale key. LITELLM_PROXY_API_KEY and --api-key precedence is unchanged, lite logout clears both stores and warns when the keychain will not release the entry, and ~/.litellm is created 0700 (tightened from 0755 where an older CLI left it broader). LITELLM_CLI_DISABLE_KEYRING=1 forces the file fallback. --- basedpyright-code-budget.json | 8 +- .../litellm_proxy_server/cli_token_usage.py | 2 +- litellm/litellm_core_utils/cli_keyring.py | 125 ++++ litellm/litellm_core_utils/cli_token_utils.py | 184 +++++- .../private_json.py | 10 + litellm/proxy/client/README.md | 12 +- litellm/proxy/client/cli/commands/agents.py | 4 +- litellm/proxy/client/cli/commands/auth.py | 148 +++-- .../client/cli/commands/claude_settings.py | 2 +- litellm/proxy/client/cli/commands/config.py | 6 +- litellm/proxy/client/cli/commands/up.py | 22 +- litellm/proxy/client/cli/main.py | 4 +- pyproject.toml | 2 + tests/test_litellm/conftest.py | 65 ++ .../test_cli_token_utils.py | 478 ++++++++++++--- .../proxy/client/cli/test_agents.py | 7 +- .../proxy/client/cli/test_auth_commands.py | 577 +++++++++--------- .../proxy/client/cli/test_claude_settings.py | 15 +- .../proxy/client/cli/test_config_commands.py | 4 +- .../proxy/client/cli/test_up_commands.py | 23 +- type-discipline-budget.json | 8 +- uv.lock | 100 ++- 22 files changed, 1295 insertions(+), 511 deletions(-) create mode 100644 litellm/litellm_core_utils/cli_keyring.py rename litellm/{proxy/client/cli/commands => litellm_core_utils}/private_json.py (64%) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 1ce71c5bd2c..59d56a3f63d 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5663 }, "reportMissingTypeArgument": { - "limit": 15557 + "limit": 15556 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 39043 + "limit": 39042 }, "reportUnknownParameterType": { - "limit": 19887 + "limit": 19886 }, "reportUnknownVariableType": { - "limit": 30574 + "limit": 30571 }, "reportUnnecessaryCast": { "limit": 117 diff --git a/cookbook/litellm_proxy_server/cli_token_usage.py b/cookbook/litellm_proxy_server/cli_token_usage.py index 6306970cdde..e6b3744019c 100644 --- a/cookbook/litellm_proxy_server/cli_token_usage.py +++ b/cookbook/litellm_proxy_server/cli_token_usage.py @@ -60,4 +60,4 @@ if __name__ == "__main__": print("\nšŸ’” Tips:") print("1. Run 'litellm-proxy login' to authenticate first") print("2. Replace 'https://your-proxy.com' with your actual proxy URL") - print("3. The token is stored locally at ~/.litellm/token.json") + print("3. The token is stored in your OS keychain, or in ~/.litellm/token.json when there is none") diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py new file mode 100644 index 00000000000..873db64a728 --- /dev/null +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -0,0 +1,125 @@ +""" +CLI Keyring Access + +SDK-level access to the OS keychain (macOS Keychain, Windows Credential Manager, +Linux Secret Service) that holds the credential minted by `lite login`. + +The `keyring` package is optional and imported lazily, so importing this module +never pulls it in. Every failure is returned as a value: a machine with no +keychain, or one whose keychain is locked, must degrade to the token file rather +than break `lite` or the SDK. +""" + +import os +from dataclasses import dataclass +from typing import Final, Protocol, TypeAlias + +KEYRING_SERVICE: Final = "litellm-cli" +KEYRING_ACCOUNT: Final = "credential" +DISABLE_KEYRING_ENV_VAR: Final = "LITELLM_CLI_DISABLE_KEYRING" + +_DISABLED_VALUES: Final = frozenset(("1", "true", "yes", "on")) + + +@dataclass(frozen=True, slots=True) +class SecretFound: + blob: str + + +@dataclass(frozen=True, slots=True) +class SecretMissing: + pass + + +@dataclass(frozen=True, slots=True) +class SecretUnavailable: + pass + + +SecretRead: TypeAlias = SecretFound | SecretMissing | SecretUnavailable + + +class SecretVault(Protocol): + """The single slot holding the CLI credential's secret material.""" + + def read(self) -> SecretRead: ... + + def write(self, blob: str) -> bool: ... + + def erase(self) -> bool: ... + + +class KeyringApi(Protocol): + def get_password(self, service_name: str, username: str) -> str | None: ... + + def set_password(self, service_name: str, username: str, password: str) -> None: ... + + def delete_password(self, service_name: str, username: str) -> None: ... + + +def _keyring_disabled() -> bool: + return os.getenv(DISABLE_KEYRING_ENV_VAR, "").strip().lower() in _DISABLED_VALUES + + +def _import_keyring() -> KeyringApi | None: + try: + import keyring + except ImportError: + return None + return keyring + + +def _keyring_api() -> KeyringApi | None: + return None if _keyring_disabled() else _import_keyring() + + +@dataclass(frozen=True, slots=True) +class KeyringVault: + """The OS keychain, reached through the optional `keyring` package.""" + + def read(self) -> SecretRead: + api: Final = _keyring_api() + if api is None: + return SecretUnavailable() + try: + blob: Final = api.get_password(KEYRING_SERVICE, KEYRING_ACCOUNT) + except Exception: # noqa: BLE001 # backends raise outside keyring.errors; never break the SDK + return SecretUnavailable() + return SecretMissing() if blob is None else SecretFound(blob) + + def write(self, blob: str) -> bool: + api: Final = _keyring_api() + if api is None: + return False + try: + api.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, blob) + except Exception: # noqa: BLE001 # a keychain that refuses the write falls back to the token file + return False + return True + + def erase(self) -> bool: + if _import_keyring() is None: + return True + if _keyring_disabled(): + # a credential stored before the kill switch was set may still be in the keychain + return False + match self.read(): + case SecretUnavailable(): + return False + case SecretMissing(): + return True + case SecretFound(): + return self._delete() + + def _delete(self) -> bool: + api: Final = _keyring_api() + if api is None: + return False + try: + api.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT) + except Exception: # noqa: BLE001 # report the failure as a value so `lite logout` can warn + return False + return True + + +SYSTEM_KEYRING: Final[SecretVault] = KeyringVault() diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index a44ce431f4e..9960192180c 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -1,17 +1,68 @@ """ CLI Token Utilities -SDK-level utilities for reading CLI authentication tokens. +SDK-level utilities for reading the credential minted by `lite login`. + +Non-secret metadata lives in ~/.litellm/token.json. The secret material (the +bearer key, plus a JWT when one is issued) lives in the OS keychain when the +machine has one, and in that same 0600 file otherwise. This module hides the +split from callers, and migrates a legacy plaintext file into the keychain the +first time it reads one. + This module has no dependencies on proxy code and can be safely imported at the SDK level. """ -import json -import os +import contextlib import time -from collections.abc import Mapping from pathlib import Path +from types import MappingProxyType from typing import Final +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.litellm_core_utils.cli_keyring import ( + SYSTEM_KEYRING, + SecretFound, + SecretMissing, + SecretUnavailable, + SecretVault, +) +from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json + + +class CliTokenRecord(BaseModel): + """A stored CLI credential. + + `key is None` means the metadata was found but the secret could not be + produced: the keychain holds nothing for us, or we could not reach it. + """ + + model_config = ConfigDict(frozen=True, extra="allow") + + base_url: str = "" + key: str | None = None + user_id: str = "" + user_email: str = "" + user_role: str = "" + auth_header_name: str = "Authorization" + jwt_token: str = "" + timestamp: float = 0.0 + + +class CliTokenSecret(BaseModel): + """The secret material as stored in the OS keychain. + + `base_url` is duplicated from the metadata file purely as a pairing tag: a + secret minted for one server is never handed to another, even if the + metadata file is edited underneath us. + """ + + model_config = ConfigDict(frozen=True) + + base_url: str + key: str + jwt_token: str = "" + def get_cli_token_file_path() -> str: """Get the path to the CLI token file""" @@ -20,26 +71,39 @@ def get_cli_token_file_path() -> str: return str(config_dir / "token.json") -def load_cli_token() -> dict | None: - """Load CLI token data from file""" - token_file: Final = get_cli_token_file_path() - if not os.path.exists(token_file): +def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | None: + """Load the stored CLI credential, or None when this machine has none""" + record: Final = _read_token_file() + if record is None: return None + return _resolve_secret(record, vault) - try: - with open(token_file, "r") as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - return None + +def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> bool: + """Store a freshly minted credential. Returns whether the keychain took the secret""" + if record.key is None or not vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)): + _write_token_file(record) + return False + _write_token_file(_without_secret(record)) + return True + + +def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> bool: + """Remove the credential from both stores. Returns whether the keychain is now free of it""" + erased: Final = vault.erase() + Path(get_cli_token_file_path()).unlink(missing_ok=True) + return erased def get_litellm_gateway_api_key( expected_base_url: str | None = None, + *, + vault: SecretVault = SYSTEM_KEYRING, ) -> str | None: """ Get the stored CLI API key for use with LiteLLM SDK. - This function reads the token file created by `lite login` + This function reads the credential created by `lite login` and returns the API key for use in Python scripts. Args: @@ -47,6 +111,7 @@ def get_litellm_gateway_api_key( originally issued for this URL. Pass the target server URL to prevent credential leakage when the client is pointed at a different (possibly malicious) server. + vault: Where the secret material is stored. Defaults to the OS keychain. Returns: str: The API key if found (and origin matches), None otherwise @@ -62,25 +127,84 @@ def get_litellm_gateway_api_key( >>> base_url="https://your-proxy.com/v1" >>> ) """ - token_data: Final = load_cli_token() - if not token_data or "key" not in token_data: + record: Final = _read_token_file() + if record is None: return None - if expected_base_url is not None: - stored_url: Final = token_data.get("base_url") - if stored_url != expected_base_url.rstrip("/"): - return None - return token_data["key"] + if expected_base_url is not None and record.base_url != expected_base_url.rstrip("/"): + return None + resolved: Final = _resolve_secret(record, vault) + return None if resolved is None else resolved.key -def is_cli_token_fresh(token_data: Mapping[str, object], buffer_hours: float = 0.1) -> bool: - """Check whether a cached CLI token (as stored in token.json) is still - within its expiration window. Used by `lite auth print-token` to fail - fast, without a network round trip, once the cached token is past - `LITELLM_CLI_JWT_EXPIRATION_HOURS`.""" +def is_cli_token_fresh(token_data: CliTokenRecord, buffer_hours: float = 0.1) -> bool: + """Check whether a cached CLI token is still within its expiration window. + Used by `lite auth print-token` to fail fast, without a network round trip, + once the cached token is past `LITELLM_CLI_JWT_EXPIRATION_HOURS`.""" from litellm.constants import CLI_JWT_EXPIRATION_HOURS - timestamp: Final = token_data.get("timestamp") - if not isinstance(timestamp, (int, float)): - return False - age_hours: Final = (time.time() - timestamp) / 3600 + age_hours: Final = (time.time() - token_data.timestamp) / 3600 return age_hours < (CLI_JWT_EXPIRATION_HOURS - buffer_hours) + + +def _read_token_file() -> CliTokenRecord | None: + try: + raw: Final = Path(get_cli_token_file_path()).read_text() + except OSError: + return None + try: + return CliTokenRecord.model_validate_json(raw) + except ValidationError: + return None + + +def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: + match vault.read(): + case SecretFound(blob=blob): + return _apply_vault_secret(record, blob, vault) + case SecretMissing(): + return _migrate_file_secret(record, vault) + case SecretUnavailable(): + return record + + +def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) -> CliTokenRecord | None: + if record.key is not None: + # a secret still on disk means the last keychain write failed: the file outranks the vault + return _migrate_file_secret(record, vault) + try: + secret: Final = CliTokenSecret.model_validate_json(blob) + except ValidationError: + return _migrate_file_secret(record, vault) + if secret.base_url != record.base_url: + return _migrate_file_secret(record, vault) + _scrub_file_secret(record) + return record.model_copy(update=MappingProxyType({"key": secret.key, "jwt_token": secret.jwt_token})) + + +def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: + if record.key is None: + return None + if vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)): + _scrub_file_secret(record) + return record + + +def _scrub_file_secret(record: CliTokenRecord) -> None: + if record.key is None and not record.jwt_token: + return + with contextlib.suppress(OSError): + _write_token_file(_without_secret(record)) + + +def _without_secret(record: CliTokenRecord) -> CliTokenRecord: + return record.model_copy(update=MappingProxyType({"key": None, "jwt_token": ""})) + + +def _encode_secret(base_url: str, key: str, jwt_token: str) -> str: + return CliTokenSecret(base_url=base_url, key=key, jwt_token=jwt_token).model_dump_json() + + +def _write_token_file(record: CliTokenRecord) -> None: + path: Final = Path(get_cli_token_file_path()) + ensure_private_dir(path.parent) + write_private_json(str(path), record.model_dump(exclude_none=True)) diff --git a/litellm/proxy/client/cli/commands/private_json.py b/litellm/litellm_core_utils/private_json.py similarity index 64% rename from litellm/proxy/client/cli/commands/private_json.py rename to litellm/litellm_core_utils/private_json.py index 31062e4a799..32bc2e169e2 100644 --- a/litellm/proxy/client/cli/commands/private_json.py +++ b/litellm/litellm_core_utils/private_json.py @@ -1,10 +1,20 @@ import json import os +import stat import tempfile from collections.abc import Mapping from pathlib import Path from typing import Final +PRIVATE_DIR_MODE: Final = 0o700 + + +def ensure_private_dir(directory: Path) -> None: + """Create directory (and parents) owner-only, tightening it if it already exists group/world readable""" + directory.mkdir(mode=PRIVATE_DIR_MODE, parents=True, exist_ok=True) + if stat.S_IMODE(directory.stat().st_mode) & 0o077: + directory.chmod(PRIVATE_DIR_MODE) + def write_private_json(path: str, data: Mapping[str, object]) -> None: """Atomically write JSON to path with owner-only permissions (0600)""" diff --git a/litellm/proxy/client/README.md b/litellm/proxy/client/README.md index 6b28f43ac73..9ece4c2be3d 100644 --- a/litellm/proxy/client/README.md +++ b/litellm/proxy/client/README.md @@ -331,7 +331,7 @@ sequenceDiagram CLI->>Proxy: Poll /sso/cli/poll/login_id with poll_secret header Proxy->>CLI: Return {"status": "ready", "key": "jwt"} - CLI->>CLI: Save key to ~/.litellm/token.json + CLI->>CLI: Save key to the OS keychain (metadata to ~/.litellm/token.json) ``` ### Authentication Commands @@ -352,7 +352,7 @@ The CLI provides these authentication commands: 5. **Callback Processing**: SSO provider redirects back to proxy with state parameter 6. **User Code Verification**: Browser confirms the verification code shown in the CLI 7. **Polling**: CLI polls `/sso/cli/poll/{login_id}` with the polling secret header until the JWT is ready. When `CLI_SSO_CLAIM_MAP` is configured on the proxy, the poll response may include `attribution_metadata` (allowlisted scalar OIDC claims for client attribution). -8. **Token Storage**: CLI saves the authentication token to `~/.litellm/token.json` +8. **Token Storage**: CLI saves the key to the OS keychain and the non-secret session metadata to `~/.litellm/token.json` ### Benefits of This Approach @@ -364,11 +364,11 @@ The CLI provides these authentication commands: ### Token Storage -Authentication tokens are stored in `~/.litellm/token.json` with restricted file permissions (600). The stored token includes: +The key itself goes into the OS keychain (macOS Keychain, Windows Credential Manager, or the Linux Secret Service) under service `litellm-cli`, account `credential`. Only the non-secret session metadata is written to `~/.litellm/token.json`, in a `0700` directory with `0600` file permissions: ```json { - "key": "sk-...", + "base_url": "https://your-proxy.com", "user_id": "cli-user", "user_email": "user@example.com", "user_role": "cli", @@ -377,6 +377,10 @@ Authentication tokens are stored in `~/.litellm/token.json` with restricted file } ``` +Headless boxes and CI runners usually have no keychain. There the key stays in the same `0600` file alongside the metadata, exactly as it did before, and `lite login` tells you which of the two happened. Set `LITELLM_CLI_DISABLE_KEYRING=1` to force the file even where a keychain exists. A `token.json` written by an older `lite` keeps working and is moved into the keychain, and scrubbed from the file, the first time a keychain-capable `lite` reads it. + +`lite logout` clears both stores. If the keychain is locked at that moment it says so, and re-running it once the keychain is unlocked finishes the job. + The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while fresh and fails once it expires -- there is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`. ### Usage diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index ed2bf2be03d..e05e85ae483 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -8,7 +8,7 @@ from typing import Final import click import requests -from .auth import get_stored_api_key, login +from .auth import context_secret_vault, get_stored_api_key, login ANTHROPIC_BASE_URL_ENV: Final = "ANTHROPIC_BASE_URL" ANTHROPIC_AUTH_TOKEN_ENV: Final = "ANTHROPIC_AUTH_TOKEN" @@ -316,7 +316,7 @@ def resolve_api_key(ctx: click.Context) -> str: click.echo("No LiteLLM credentials found; starting login...") ctx.invoke(login) - api_key = get_stored_api_key(expected_base_url=base_url) + api_key = get_stored_api_key(expected_base_url=base_url, vault=context_secret_vault(ctx)) if not api_key: raise click.ClickException("Login did not produce an API key; cannot start the agent.") return api_key diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 0a0bcf80ee5..8b9ef5633da 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -1,9 +1,6 @@ -import json -import os import sys import time import webbrowser -from pathlib import Path from typing import Any, Final from urllib.parse import urlencode @@ -11,10 +8,19 @@ import click import requests from rich.console import Console from rich.table import Table -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import CLI_JWT_EXPIRATION_HOURS -from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh +from litellm.litellm_core_utils.cli_keyring import SYSTEM_KEYRING, SecretVault +from litellm.litellm_core_utils.cli_token_utils import ( + CliTokenRecord, + clear_cli_token, + get_cli_token_file_path, + get_litellm_gateway_api_key, + is_cli_token_fresh, + load_cli_token, + save_cli_token, +) from .claude_settings import ( CLAUDE_SETTINGS_PATH, @@ -22,18 +28,6 @@ from .claude_settings import ( ClaudeSettingsError, write_claude_settings, ) -from .private_json import write_private_json - - -class CliTokenData(TypedDict): - base_url: str - key: str - user_id: str - user_email: str - user_role: str - auth_header_name: str - jwt_token: str - timestamp: float class CliTeam(TypedDict, total=False): @@ -46,6 +40,7 @@ class CliTeam(TypedDict, total=False): class CliContextObj(TypedDict): base_url: str base_url_explicit: NotRequired[bool] + secret_vault: NotRequired[ReadOnly[SecretVault]] class CliPollData(TypedDict, total=False): @@ -76,50 +71,32 @@ class CliAuthResult(TypedDict): team_id: str | None +KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( + "Your credential is stored in your OS keychain, which could not be read. Unlock it and retry, or run 'lite login'." +) + + # Token storage utilities -def get_token_file_path() -> str: - """Get the path to store the authentication token""" - home_dir: Final = Path.home() - config_dir: Final = home_dir / ".litellm" - config_dir.mkdir(exist_ok=True) - return str(config_dir / "token.json") +def context_secret_vault(ctx: click.Context) -> SecretVault: + """Where this invocation reads and writes secret material; injectable through ctx.obj for tests""" + ctx_obj: Final[CliContextObj | None] = ctx.obj + if ctx_obj is None: + return SYSTEM_KEYRING + return ctx_obj.get("secret_vault") or SYSTEM_KEYRING -def save_token(token_data: CliTokenData) -> None: - """Save token data to file""" - write_private_json(get_token_file_path(), token_data) - - -def load_token() -> CliTokenData | None: - """Load token data from file""" - token_file: Final = get_token_file_path() - if not os.path.exists(token_file): - return None - - try: - with open(token_file, "r") as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - return None - - -def clear_token() -> None: - """Clear stored token""" - token_file: Final = get_token_file_path() - if os.path.exists(token_file): - os.remove(token_file) - - -def get_stored_api_key(expected_base_url: str | None = None) -> str | None: - """Get the stored API key from token file. +def get_stored_api_key( + expected_base_url: str | None = None, + *, + vault: SecretVault = SYSTEM_KEYRING, +) -> str | None: + """Get the stored API key. If expected_base_url is provided, the key is only returned when it was originally issued for that URL. This prevents credential leakage when the CLI is pointed at a different (possibly malicious) server. """ - from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key - - return get_litellm_gateway_api_key(expected_base_url=expected_base_url) + return get_litellm_gateway_api_key(expected_base_url=expected_base_url, vault=vault) # Team selection utilities @@ -689,23 +666,27 @@ def login(ctx: click.Context, config_claude: bool): api_key: Final = auth_result["api_key"] user_id: Final = auth_result["user_id"] - # Save token data. base_url is stored so we can verify origin - # before reusing the key on a subsequent CLI invocation. - save_token( - { - "base_url": base_url.rstrip("/"), - "key": api_key, - "user_id": user_id or "cli-user", - "user_email": "unknown", - "user_role": "cli", - "auth_header_name": "Authorization", - "jwt_token": "", - "timestamp": time.time(), - } + # base_url is stored so we can verify origin before reusing the + # key on a subsequent CLI invocation. + record: Final = CliTokenRecord( + base_url=base_url.rstrip("/"), + key=api_key, + user_id=user_id or "cli-user", + user_email="unknown", + user_role="cli", + auth_header_name="Authorization", + jwt_token="", + timestamp=time.time(), ) + in_keychain: Final = save_cli_token(record, vault=context_secret_vault(ctx)) click.echo("\nLogin successful!") click.echo(f"JWT Token: {api_key[:20]}...") + click.echo( + "Credential stored in your OS keychain." + if in_keychain + else f"No OS keychain available; credential stored in {get_cli_token_file_path()} (owner-only)." + ) click.echo("You can now use the CLI without specifying --api-key") if config_claude: @@ -736,10 +717,14 @@ def login(ctx: click.Context, config_claude: bool): @click.command(name="logout") -def logout(): +@click.pass_context +def logout(ctx: click.Context): """Logout and clear stored authentication""" - clear_token() - click.echo("Logged out successfully. Authentication token cleared.") + if clear_cli_token(vault=context_secret_vault(ctx)): + click.echo("Logged out successfully. Authentication token cleared.") + return + click.echo("Logged out. The local token file is gone, but the OS keychain entry could not be removed.") + click.echo("Unlock your keychain and run 'lite logout' again to clear it.") @click.command(name="print-token") @@ -753,7 +738,7 @@ def print_token(ctx: click.Context): expires after `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h); once expired, run `lite login` again. """ - token_data: Final = load_token() + token_data: Final = load_cli_token(vault=context_secret_vault(ctx)) if not token_data: click.echo("Not authenticated. Run 'lite login'.", err=True) sys.exit(1) @@ -765,7 +750,7 @@ def print_token(ctx: click.Context): ctx_obj: Final[CliContextObj] = ctx.obj if ctx_obj.get("base_url_explicit"): base_url: Final = ctx_obj["base_url"] - if token_data.get("base_url") != base_url.rstrip("/"): + if token_data.base_url != base_url.rstrip("/"): click.echo("Not authenticated for this server. Run 'lite login'.", err=True) sys.exit(1) @@ -773,33 +758,36 @@ def print_token(ctx: click.Context): click.echo("Token expired. Run 'lite login' again.", err=True) sys.exit(1) - api_key: Final = token_data.get("key") + api_key: Final = token_data.key if not api_key: - click.echo("No token available. Run 'lite login'.", err=True) + click.echo(KEYCHAIN_UNREACHABLE_MESSAGE, err=True) sys.exit(1) click.echo(api_key) @click.command(name="whoami") -def whoami(): +@click.pass_context +def whoami(ctx: click.Context): """Show current authentication status""" - token_data: Final = load_token() + token_data: Final = load_cli_token(vault=context_secret_vault(ctx)) if not token_data: click.echo("Not authenticated. Run 'lite login' to authenticate.") return click.echo("Authenticated") - click.echo(f"User Email: {token_data.get('user_email', 'Unknown')}") - click.echo(f"User ID: {token_data.get('user_id', 'Unknown')}") - click.echo(f"User Role: {token_data.get('user_role', 'Unknown')}") + click.echo(f"User Email: {token_data.user_email or 'Unknown'}") + click.echo(f"User ID: {token_data.user_id or 'Unknown'}") + click.echo(f"User Role: {token_data.user_role or 'Unknown'}") # Check if token is still valid (basic timestamp check) - timestamp: Final = token_data.get("timestamp", 0) - age_hours: Final = (time.time() - timestamp) / 3600 + age_hours: Final = (time.time() - token_data.timestamp) / 3600 click.echo(f"Token age: {age_hours:.1f} hours") + if token_data.key is None: + click.echo(KEYCHAIN_UNREACHABLE_MESSAGE) + if age_hours > CLI_JWT_EXPIRATION_HOURS: click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.") diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index e9a6a25a064..e18e5b1b7ee 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -15,7 +15,7 @@ from typing import Final from pydantic import JsonValue, TypeAdapter, ValidationError -from .private_json import write_private_json +from litellm.litellm_core_utils.private_json import write_private_json ENV_KEY: Final = "env" API_KEY_HELPER_KEY: Final = "apiKeyHelper" diff --git a/litellm/proxy/client/cli/commands/config.py b/litellm/proxy/client/cli/commands/config.py index 19dd407ba19..2715a0a9a38 100644 --- a/litellm/proxy/client/cli/commands/config.py +++ b/litellm/proxy/client/cli/commands/config.py @@ -10,7 +10,7 @@ from urllib.parse import urlparse import click from pydantic import TypeAdapter -from .private_json import write_private_json +from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json HIDDEN_COMMANDS_KEY: Final = "hidden_commands" @@ -42,7 +42,9 @@ def load_config() -> Mapping[str, str]: def save_config(config: Mapping[str, str]) -> None: """Save CLI config to file""" - write_private_json(get_config_file_path(), config) + config_file: Final = Path(get_config_file_path()) + ensure_private_dir(config_file.parent) + write_private_json(str(config_file), config) def get_config_value(key: str) -> str | None: diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index dd266b4afa1..a0fd4af8f72 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -14,10 +14,12 @@ from typing import IO, Final import click from pydantic import JsonValue, TypeAdapter, ValidationError -from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh +from litellm.litellm_core_utils.cli_keyring import SecretVault +from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh, load_cli_token +from litellm.litellm_core_utils.private_json import ensure_private_dir from .agents import AgentRunError, resolve_api_key, verify_proxy_key -from .auth import load_token, login +from .auth import context_secret_vault, login from .claude_settings import ( BACKUP_PATH, CLAUDE_SETTINGS_PATH, @@ -66,7 +68,7 @@ def secure_create(path: Path) -> Iterator[IO[str]]: def write_backup(record: BackupRecord, backup_path: Path | None = None) -> None: path: Final = backup_path if backup_path is not None else BACKUP_PATH - path.parent.mkdir(exist_ok=True) + ensure_private_dir(path.parent) with secure_create(path) as f: json.dump({"existed": record.existed, "content": record.content}, f, indent=2) @@ -103,10 +105,17 @@ def restore_claude_settings(settings_path: Path | None = None, backup_path: Path return record +def _has_fresh_login(base_url: str, vault: SecretVault) -> bool: + token_data: Final = load_cli_token(vault=vault) + if token_data is None or token_data.key is None or token_data.base_url != base_url: + return False + return is_cli_token_fresh(token_data) + + def _ensure_fresh_login(ctx: click.Context) -> None: base_url: Final = ctx.obj["base_url"].rstrip("/") - token_data = load_token() - if token_data and token_data.get("base_url") == base_url and is_cli_token_fresh(token_data): + vault: Final = context_secret_vault(ctx) + if _has_fresh_login(base_url, vault): return if not sys.stdin.isatty(): @@ -117,8 +126,7 @@ def _ensure_fresh_login(ctx: click.Context) -> None: click.echo("No fresh LiteLLM login found for this proxy; starting login...") ctx.invoke(login) - token_data = load_token() - if not token_data or token_data.get("base_url") != base_url or not is_cli_token_fresh(token_data): + if not _has_fresh_login(base_url, vault): raise UpError("Login did not produce a usable token; cannot start `lite up`.") diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 3a289736c66..664bf5a216c 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -9,7 +9,7 @@ from litellm._version import version as litellm_version from litellm.proxy.client.health import HealthManagementClient from .commands.agents import agent_commands -from .commands.auth import auth_group, get_stored_api_key, login, logout, whoami +from .commands.auth import auth_group, context_secret_vault, get_stored_api_key, login, logout, whoami from .commands.autoroute.commands import autoroute_group from .commands.chat import chat from .commands.config import config_commands, get_config_value, hidden_command_names @@ -94,7 +94,7 @@ def cli(ctx: click.Context, show_version: bool, base_url: str | None, api_key: s # If no API key provided via flag or environment variable, try to load from saved token. # Pass base_url so we only use the stored key when it was issued for this server. if api_key is None: - api_key = get_stored_api_key(expected_base_url=base_url) + api_key = get_stored_api_key(expected_base_url=base_url, vault=context_secret_vault(ctx)) ctx.obj["base_url"] = base_url ctx.obj["api_key"] = api_key diff --git a/pyproject.toml b/pyproject.toml index ffbc96eefb9..32921e14d31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ cli = [ "pyyaml>=6.0.3,<7.0", "requests>=2.32.0,<3.0", "InquirerPy>=0.3.4,<1.0", + "keyring>=25.6.0,<26.0", ] extra_proxy = [ "prisma>=0.11.0,<1.0", @@ -166,6 +167,7 @@ litellm-proxy = "litellm.proxy.client.cli:cli" dev = [ "diff-cover==9.7.2", "basedpyright==1.39.7", + "keyring==25.7.0", "pytest==9.0.3", "pytest-mock==3.15.1", "pytest-asyncio==1.3.0", diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index c0644c88291..ce0fd197538 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -22,6 +22,12 @@ import litellm from litellm import router as litellm_router_module from litellm import utils as litellm_utils_module from litellm._logging import ALL_LOGGERS +from litellm.litellm_core_utils.cli_keyring import ( + SecretFound, + SecretMissing, + SecretRead, + SecretUnavailable, +) from litellm.litellm_core_utils.prompt_templates import ( image_handling as image_handling_module, ) @@ -106,6 +112,65 @@ def isolate_host_proxy_base_url(monkeypatch): monkeypatch.delenv("PROXY_BASE_URL", raising=False) +@pytest.fixture(scope="function", autouse=True) +def isolate_host_os_keychain(monkeypatch): + """Keep any code path that resolves a CLI credential out of the developer's real OS keychain. + + Tests that exercise keychain behaviour inject their own vault instead. + """ + monkeypatch.setenv("LITELLM_CLI_DISABLE_KEYRING", "1") + + +class FakeSecretVault: + """In-memory stand-in for the OS keychain, injected wherever CLI credential storage is exercised. + + `available=False` models a keychain that is locked or has no backend, `writable=False` one that + refuses to store, and `erasable=False` one that will not release what it already holds. + """ + + def __init__( + self, + blob: str | None = None, + *, + available: bool = True, + writable: bool = True, + erasable: bool = True, + ) -> None: + self.blob: str | None = blob + self.available: bool = available + self.writable: bool = writable + self.erasable: bool = erasable + self.reads: int = 0 + self.writes: list[str] = [] + self.erases: int = 0 + + def read(self) -> SecretRead: + self.reads += 1 + if not self.available: + return SecretUnavailable() + return SecretMissing() if self.blob is None else SecretFound(self.blob) + + def write(self, blob: str) -> bool: + self.writes.append(blob) + if not (self.available and self.writable): + return False + self.blob = blob + return True + + def erase(self) -> bool: + self.erases += 1 + if not (self.available and self.erasable): + return False + self.blob = None + return True + + +@pytest.fixture +def secret_vault_factory(): + """Build FakeSecretVault instances; see its docstring for the failure modes it can model.""" + return FakeSecretVault + + def _run_coroutine_if_needed(result): if not asyncio.iscoroutine(result): return diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 27fc5eb4bd0..56ab6bcbfe0 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -1,89 +1,429 @@ -""" -Unit tests for CLI token utilities -""" - import json -import os -import tempfile -from pathlib import Path -from unittest.mock import mock_open, patch +import stat +import sys +import time import pytest -from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key +from litellm.constants import CLI_JWT_EXPIRATION_HOURS +from litellm.litellm_core_utils.cli_keyring import ( + DISABLE_KEYRING_ENV_VAR, + KEYRING_ACCOUNT, + KEYRING_SERVICE, + KeyringVault, + SecretFound, + SecretMissing, + SecretUnavailable, +) +from litellm.litellm_core_utils.cli_token_utils import ( + CliTokenRecord, + clear_cli_token, + get_cli_token_file_path, + get_litellm_gateway_api_key, + is_cli_token_fresh, + load_cli_token, + save_cli_token, +) + +SERVER = "https://proxy.example.com" +OTHER_SERVER = "https://other-proxy.example.com" -class TestCLITokenUtils: - """Test CLI token utility functions""" +@pytest.fixture +def isolated_home(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + return tmp_path - def test_get_litellm_gateway_api_key_success(self): - """Test getting CLI API key when token file exists and is valid""" - token_data = { - "key": "sk-test-cli-key-123", - "user_id": "test-user", - "user_email": "test@example.com", - "timestamp": 1234567890, - } - with ( - patch("os.path.exists", return_value=True), - patch("builtins.open", mock_open(read_data=json.dumps(token_data))), - patch( - "litellm.litellm_core_utils.cli_token_utils.get_cli_token_file_path", - return_value="/test/.litellm/token.json", - ), - ): +def _token_file(home): + return home / ".litellm" / "token.json" - result = get_litellm_gateway_api_key() - assert result == "sk-test-cli-key-123" +def _write_legacy_file(home, **overrides): + payload = { + "base_url": SERVER, + "key": "sk-legacy", + "user_id": "u-1", + "user_email": "user@example.com", + "user_role": "cli", + "timestamp": time.time(), + **overrides, + } + path = _token_file(home) + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(payload)) + path.chmod(0o600) + return path - def test_get_litellm_gateway_api_key_no_file(self): - """Test getting CLI API key when token file doesn't exist""" - with ( - patch("os.path.exists", return_value=False), - patch( - "litellm.litellm_core_utils.cli_token_utils.get_cli_token_file_path", - return_value="/test/.litellm/token.json", - ), - ): - result = get_litellm_gateway_api_key() +def _write_metadata_only_file(home): + """What a post-migration token.json looks like: everything except the secret material.""" + path = _token_file(home) + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps({"base_url": SERVER, "user_id": "u-1", "timestamp": time.time()})) + path.chmod(0o600) + return path - assert result is None - def test_get_litellm_gateway_api_key_invalid_json(self): - """Test getting CLI API key when token file has invalid JSON""" - with ( - patch("os.path.exists", return_value=True), - patch("builtins.open", mock_open(read_data="invalid json")), - patch( - "litellm.litellm_core_utils.cli_token_utils.get_cli_token_file_path", - return_value="/test/.litellm/token.json", - ), - ): +def _blob(base_url=SERVER, key="sk-vault", jwt_token=""): + return json.dumps({"base_url": base_url, "key": key, "jwt_token": jwt_token}) - result = get_litellm_gateway_api_key() - assert result is None +class TestGetCliTokenFilePath: + def test_points_at_the_home_config_file(self, isolated_home): + assert get_cli_token_file_path() == str(isolated_home / ".litellm" / "token.json") - def test_get_litellm_gateway_api_key_no_key_field(self): - """Test getting CLI API key when token file exists but has no key field""" - token_data = { - "user_id": "test-user", - "user_email": "test@example.com", - # Missing 'key' field - } + def test_does_not_create_the_directory(self, isolated_home): + """Merely asking for the path must not leave a directory behind, so an SDK import that + never logs in cannot create a ~/.litellm on someone's machine.""" + get_cli_token_file_path() - with ( - patch("os.path.exists", return_value=True), - patch("builtins.open", mock_open(read_data=json.dumps(token_data))), - patch( - "litellm.litellm_core_utils.cli_token_utils.get_cli_token_file_path", - return_value="/test/.litellm/token.json", - ), - ): + assert not (isolated_home / ".litellm").exists() - result = get_litellm_gateway_api_key() - assert result is None +class TestLoadCliToken: + def test_no_token_file_never_touches_the_keychain(self, isolated_home, secret_vault_factory): + """The SDK calls this on machines that never ran `lite login`; it must not prompt for + keychain access there.""" + vault = secret_vault_factory(blob=_blob()) + + assert load_cli_token(vault=vault) is None + assert vault.reads == 0 + + def test_secret_comes_from_the_vault_when_the_file_holds_only_metadata(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(blob=_blob(key="sk-from-keychain")) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-from-keychain" + assert "sk-from-keychain" not in _token_file(isolated_home).read_text() + + def test_jwt_token_round_trips_through_the_vault(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(blob=_blob(key="sk-a", jwt_token="jwt-a")) + + record = load_cli_token(vault=vault) + + assert (record.key, record.jwt_token) == ("sk-a", "jwt-a") + + def test_legacy_plaintext_file_still_authenticates_and_is_migrated(self, isolated_home, secret_vault_factory): + """A token.json written by an older `lite` keeps working, and reading it moves the secret + into the keychain and scrubs it from disk.""" + path = _write_legacy_file(isolated_home) + vault = secret_vault_factory() + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert json.loads(vault.blob)["key"] == "sk-legacy" + on_disk = json.loads(path.read_text()) + assert "key" not in on_disk + assert on_disk["user_email"] == "user@example.com" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + def test_legacy_file_survives_a_vault_that_refuses_to_store(self, isolated_home, secret_vault_factory): + """Scrubbing the only copy of the secret after a failed keychain write would log the user + out for good.""" + path = _write_legacy_file(isolated_home) + before = path.read_text() + + record = load_cli_token(vault=secret_vault_factory(writable=False)) + + assert record.key == "sk-legacy" + assert path.read_text() == before + + def test_a_secret_left_on_disk_outranks_a_stale_keychain_entry(self, isolated_home, secret_vault_factory): + """A failed keychain write leaves the fresh secret on disk while the vault still holds the + previous one; the next read must serve the file's secret and move it into the vault, never + resurrect the stale key or scrub the only copy of the fresh one.""" + path = _write_legacy_file(isolated_home, key="sk-fresh") + vault = secret_vault_factory(blob=_blob(key="sk-stale")) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-fresh" + assert json.loads(vault.blob)["key"] == "sk-fresh" + assert "key" not in json.loads(path.read_text()) + + def test_a_disk_secret_survives_when_the_stale_vault_refuses_the_rewrite( + self, isolated_home, secret_vault_factory + ): + path = _write_legacy_file(isolated_home, key="sk-fresh") + before = path.read_text() + + record = load_cli_token(vault=secret_vault_factory(blob=_blob(key="sk-stale"), writable=False)) + + assert record.key == "sk-fresh" + assert path.read_text() == before + + def test_legacy_file_survives_an_unreachable_vault_without_write_attempts( + self, isolated_home, secret_vault_factory + ): + path = _write_legacy_file(isolated_home) + before = path.read_text() + vault = secret_vault_factory(available=False) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert vault.writes == [] + assert path.read_text() == before + + def test_metadata_only_file_with_an_empty_vault_is_not_a_login(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + + assert load_cli_token(vault=secret_vault_factory()) is None + + def test_metadata_only_file_with_an_unreachable_vault_reports_a_missing_secret( + self, isolated_home, secret_vault_factory + ): + """The caller needs to tell "never logged in" apart from "locked keychain", so the record + comes back with no key rather than as None.""" + _write_metadata_only_file(isolated_home) + + record = load_cli_token(vault=secret_vault_factory(available=False)) + + assert record.key is None + assert record.user_id == "u-1" + + def test_a_secret_minted_for_another_server_is_never_handed_out(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + + assert load_cli_token(vault=secret_vault_factory(blob=_blob(base_url=OTHER_SERVER))) is None + + def test_a_secret_minted_for_another_server_loses_to_the_file(self, isolated_home, secret_vault_factory): + _write_legacy_file(isolated_home) + vault = secret_vault_factory(blob=_blob(base_url=OTHER_SERVER, key="sk-elsewhere")) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert json.loads(vault.blob)["key"] == "sk-legacy" + + def test_unreadable_vault_blob_falls_back_to_the_file_secret(self, isolated_home, secret_vault_factory): + _write_legacy_file(isolated_home) + + record = load_cli_token(vault=secret_vault_factory(blob="not json at all {{{")) + + assert record.key == "sk-legacy" + + def test_corrupt_token_file_is_not_a_login(self, isolated_home, secret_vault_factory): + _token_file(isolated_home).parent.mkdir() + _token_file(isolated_home).write_text("not json at all {{{") + + assert load_cli_token(vault=secret_vault_factory(blob=_blob())) is None + + +class TestGetLitellmGatewayApiKey: + def test_returns_the_vault_secret_when_the_origin_matches(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + + key = get_litellm_gateway_api_key(expected_base_url=SERVER, vault=secret_vault_factory(blob=_blob())) + + assert key == "sk-vault" + + def test_trailing_slash_on_the_expected_url_is_normalised(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + + key = get_litellm_gateway_api_key(expected_base_url=SERVER + "/", vault=secret_vault_factory(blob=_blob())) + + assert key == "sk-vault" + + def test_origin_mismatch_returns_nothing_without_reading_the_keychain(self, isolated_home, secret_vault_factory): + """Pointing the SDK at a different server must fail before the keychain is even consulted, + so a hostile base_url cannot provoke an unlock prompt.""" + _write_legacy_file(isolated_home) + vault = secret_vault_factory(blob=_blob()) + + assert get_litellm_gateway_api_key(expected_base_url=OTHER_SERVER, vault=vault) is None + assert vault.reads == 0 + + def test_no_token_file_returns_nothing(self, isolated_home, secret_vault_factory): + assert get_litellm_gateway_api_key(vault=secret_vault_factory(blob=_blob())) is None + + +class TestSaveCliToken: + def test_secret_goes_to_the_keychain_and_never_to_the_file(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + + stored = save_cli_token( + CliTokenRecord(base_url=SERVER, key="sk-new", user_id="u-1", timestamp=time.time()), + vault=vault, + ) + + assert stored is True + assert "sk-new" not in _token_file(isolated_home).read_text() + assert json.loads(vault.blob)["key"] == "sk-new" + assert load_cli_token(vault=vault).key == "sk-new" + + def test_falls_back_to_the_owner_only_file_when_there_is_no_keychain(self, isolated_home, secret_vault_factory): + stored = save_cli_token( + CliTokenRecord(base_url=SERVER, key="sk-new", timestamp=time.time()), + vault=secret_vault_factory(available=False), + ) + + path = _token_file(isolated_home) + assert stored is False + assert json.loads(path.read_text())["key"] == "sk-new" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert list(path.parent.glob(".tmp-*")) == [] + + def test_creates_the_config_directory_owner_only(self, isolated_home, secret_vault_factory): + """A 0755 ~/.litellm lets any local process list, and in the fallback case read, the + credential's directory.""" + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=secret_vault_factory()) + + assert stat.S_IMODE((isolated_home / ".litellm").stat().st_mode) == 0o700 + + def test_tightens_a_directory_left_group_readable_by_an_older_cli(self, isolated_home, secret_vault_factory): + config_dir = isolated_home / ".litellm" + config_dir.mkdir(mode=0o755) + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=secret_vault_factory()) + + assert stat.S_IMODE(config_dir.stat().st_mode) == 0o700 + + def test_a_failed_write_leaves_the_previous_credential_intact(self, isolated_home, secret_vault_factory, monkeypatch): + path = _write_legacy_file(isolated_home) + before = path.read_text() + + def _explode(*args, **kwargs): + raise TypeError("not serialisable") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + + with pytest.raises(TypeError): + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=secret_vault_factory(available=False)) + + assert path.read_text() == before + assert list(path.parent.glob(".tmp-*")) == [] + + +class TestClearCliToken: + def test_removes_the_credential_from_both_stores(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) + + assert clear_cli_token(vault=vault) is True + assert vault.blob is None + assert not _token_file(isolated_home).exists() + assert load_cli_token(vault=vault) is None + + def test_reports_a_keychain_that_will_not_release_the_secret(self, isolated_home, secret_vault_factory): + _write_legacy_file(isolated_home) + vault = secret_vault_factory(blob=_blob(), erasable=False) + + assert clear_cli_token(vault=vault) is False + assert not _token_file(isolated_home).exists() + + def test_is_safe_when_nothing_was_ever_stored(self, isolated_home, secret_vault_factory): + assert clear_cli_token(vault=secret_vault_factory()) is True + + +class TestIsCliTokenFresh: + def test_a_just_issued_token_is_fresh(self): + assert is_cli_token_fresh(CliTokenRecord(timestamp=time.time())) is True + + def test_a_token_past_its_expiry_is_stale(self): + stale = CliTokenRecord(timestamp=time.time() - (CLI_JWT_EXPIRATION_HOURS + 1) * 3600) + + assert is_cli_token_fresh(stale) is False + + def test_the_buffer_retires_a_token_just_before_it_expires(self): + almost = CliTokenRecord(timestamp=time.time() - (CLI_JWT_EXPIRATION_HOURS * 3600 - 60)) + + assert is_cli_token_fresh(almost, buffer_hours=0.1) is False + + +class _FakeKeyringModule: + def __init__(self, stored=None, *, get_error=None, set_error=None, delete_error=None): + self.stored = stored + self.get_error = get_error + self.set_error = set_error + self.delete_error = delete_error + self.calls = [] + + def get_password(self, service_name, username): + self.calls.append(("get", service_name, username)) + if self.get_error is not None: + raise self.get_error + return self.stored + + def set_password(self, service_name, username, password): + self.calls.append(("set", service_name, username)) + if self.set_error is not None: + raise self.set_error + self.stored = password + + def delete_password(self, service_name, username): + self.calls.append(("delete", service_name, username)) + if self.delete_error is not None: + raise self.delete_error + self.stored = None + + +@pytest.fixture +def install_fake_keyring(monkeypatch): + def _install(fake): + monkeypatch.delenv(DISABLE_KEYRING_ENV_VAR, raising=False) + monkeypatch.setitem(sys.modules, "keyring", fake) + return fake + + return _install + + +class TestKeyringVault: + def test_round_trips_through_the_installed_keyring(self, install_fake_keyring): + fake = install_fake_keyring(_FakeKeyringModule()) + vault = KeyringVault() + + assert vault.write("blob-1") is True + assert vault.read() == SecretFound("blob-1") + assert vault.erase() is True + assert vault.read() == SecretMissing() + assert {call[1:] for call in fake.calls} == {(KEYRING_SERVICE, KEYRING_ACCOUNT)} + + def test_the_kill_switch_reports_no_keychain(self, monkeypatch): + """`LITELLM_CLI_DISABLE_KEYRING` has to work without importing keyring, because keyring + caches its backend on first use and cannot be reconfigured later. Erase still fails: a + credential stored before the switch was set may be in the keychain, and with reads + disabled `lite logout` cannot verify it is gone, so it must warn instead.""" + monkeypatch.setenv(DISABLE_KEYRING_ENV_VAR, "1") + vault = KeyringVault() + + assert vault.read() == SecretUnavailable() + assert vault.write("blob-1") is False + assert vault.erase() is False + + def test_an_uninstalled_keyring_library_degrades_to_the_file(self, monkeypatch): + """keyring is an optional extra, so the SDK must survive its absence rather than raise on + the hot path.""" + monkeypatch.delenv(DISABLE_KEYRING_ENV_VAR, raising=False) + monkeypatch.setitem(sys.modules, "keyring", None) + vault = KeyringVault() + + assert vault.read() == SecretUnavailable() + assert vault.write("blob-1") is False + assert vault.erase() is True + + def test_a_locked_keychain_is_reported_not_raised(self, install_fake_keyring): + install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("keyring is locked"))) + + assert KeyringVault().read() == SecretUnavailable() + + def test_a_refused_write_is_reported_not_raised(self, install_fake_keyring): + install_fake_keyring(_FakeKeyringModule(set_error=RuntimeError("no backend"))) + + assert KeyringVault().write("blob-1") is False + + def test_a_refused_delete_is_reported_so_logout_can_warn(self, install_fake_keyring): + install_fake_keyring(_FakeKeyringModule(stored="blob-1", delete_error=RuntimeError("locked"))) + + assert KeyringVault().erase() is False + + def test_erasing_a_locked_keychain_is_a_failure(self, install_fake_keyring): + install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("locked"))) + + assert KeyringVault().erase() is False diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py index a23c573047f..c2858c84c6d 100644 --- a/tests/test_litellm/proxy/client/cli/test_agents.py +++ b/tests/test_litellm/proxy/client/cli/test_agents.py @@ -672,8 +672,9 @@ class TestAgentCommands: assert "LITELLM_PROXY_API_KEY" in result.output mock_run.assert_not_called() - def test_interactive_without_key_logs_in_then_launches(self): + def test_interactive_without_key_logs_in_then_launches(self, secret_vault_factory): captured = {} + vault = secret_vault_factory() @click.command() def fake_login(): @@ -695,12 +696,12 @@ class TestAgentCommands: result = self.runner.invoke( _agent_command("claude"), [], - obj={"base_url": "http://localhost:4000", "api_key": None}, + obj={"base_url": "http://localhost:4000", "api_key": None, "secret_vault": vault}, ) assert result.exit_code == 0, result.output assert captured["api_key"] == "sk-after-login" - mock_get.assert_called_once_with(expected_base_url="http://localhost:4000") + mock_get.assert_called_once_with(expected_base_url="http://localhost:4000", vault=vault) def test_child_exit_code_reaches_the_shell(self): with patch(f"{AGENTS_MODULE}.run_agent", side_effect=SystemExit(42)): diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 59048067674..e93f05cb4aa 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -4,7 +4,7 @@ import stat import sys import time from pathlib import Path -from unittest.mock import Mock, mock_open, patch +from unittest.mock import Mock, patch sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path @@ -13,21 +13,38 @@ import pytest from click.testing import CliRunner from litellm.constants import CLI_JWT_EXPIRATION_HOURS +from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord, save_cli_token from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.auth import ( - clear_token, + KEYCHAIN_UNREACHABLE_MESSAGE, get_stored_api_key, - get_token_file_path, - load_token, login, logout, print_token, - save_token, whoami, ) from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner +@pytest.fixture +def isolated_home(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.delenv("LITELLM_PROXY_URL", raising=False) + monkeypatch.delenv("LITELLM_PROXY_API_KEY", raising=False) + return tmp_path + + +def _write_home_json(home: Path, filename: str, payload: dict[str, object]) -> None: + litellm_dir = home / ".litellm" + litellm_dir.mkdir(exist_ok=True) + (litellm_dir / filename).write_text(json.dumps(payload)) + + +def _secret_blob(base_url: str, key: str) -> str: + return json.dumps({"base_url": base_url, "key": key, "jwt_token": ""}) + + def _mock_cli_sso_start_response( login_id: str = "cli-session-uuid-456", poll_secret: str = "poll-secret", @@ -176,200 +193,50 @@ class TestStartCliSsoFlowErrors: assert "https://unreachable.example.com/sso/cli/start" in message -class TestTokenUtilities: - """Test token file utility functions""" +class TestStoredApiKeyLookup: + """`get_stored_api_key` is what every other `lite` subcommand authenticates with, so the + keychain split and the origin check both have to be invisible to it.""" - def test_get_token_file_path(self): - """Test getting token file path""" - with ( - patch("pathlib.Path.home") as mock_home, - patch("pathlib.Path.mkdir") as mock_mkdir, - ): - mock_home.return_value = Path("/home/user") + def test_returns_the_secret_the_keychain_holds(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "user_id": "u-1"}) + vault = secret_vault_factory(blob=_secret_blob("https://real-proxy.com", "sk-from-keychain")) - result = get_token_file_path() + assert get_stored_api_key(vault=vault) == "sk-from-keychain" - assert result == "/home/user/.litellm/token.json" - mock_mkdir.assert_called_once_with(exist_ok=True) + def test_returns_a_legacy_plaintext_key(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "key": "sk-legacy"}) - def test_get_token_file_path_creates_directory(self): - """Test that get_token_file_path creates the config directory""" - with ( - patch("pathlib.Path.home") as mock_home, - patch("pathlib.Path.mkdir") as mock_mkdir, - ): - mock_home.return_value = Path("/home/user") + assert get_stored_api_key(vault=secret_vault_factory()) == "sk-legacy" - get_token_file_path() + def test_no_token_at_all_returns_nothing(self, isolated_home, secret_vault_factory): + assert get_stored_api_key(vault=secret_vault_factory()) is None - mock_mkdir.assert_called_once_with(exist_ok=True) + def test_metadata_without_a_secret_returns_nothing(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "user_id": "u-1"}) - def test_save_token(self, tmp_path): - """Test saving token data to file""" - token_data = { - "key": "test-key", - "user_id": "test-user", - "timestamp": 1234567890, - } - token_file = tmp_path / "token.json" + assert get_stored_api_key(vault=secret_vault_factory()) is None - with patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path: - mock_path.return_value = str(token_file) + def test_matching_base_url_returns_the_key(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "key": "sk-prod"}) - save_token(token_data) + assert get_stored_api_key("https://real-proxy.com", vault=secret_vault_factory()) == "sk-prod" - assert json.loads(token_file.read_text()) == token_data - assert stat.S_IMODE(token_file.stat().st_mode) == 0o600 + def test_trailing_slash_on_the_expected_url_is_normalised(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "key": "sk-prod"}) - def test_load_token_success(self): - """Test loading token data from file successfully""" - token_data = { - "key": "test-key", - "user_id": "test-user", - "timestamp": 1234567890, - } + assert get_stored_api_key("https://real-proxy.com/", vault=secret_vault_factory()) == "sk-prod" - with ( - patch("builtins.open", mock_open(read_data=json.dumps(token_data))), - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=True), - ): - mock_path.return_value = "/test/path/token.json" + def test_mismatched_base_url_withholds_the_key(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "key": "sk-prod"}) - result = load_token() + assert get_stored_api_key("https://evil.com", vault=secret_vault_factory()) is None - assert result == token_data + def test_old_tokens_without_a_base_url_are_rejected_when_an_origin_is_expected( + self, isolated_home, secret_vault_factory + ): + _write_home_json(isolated_home, "token.json", {"key": "sk-old-token"}) - def test_load_token_file_not_exists(self): - """Test loading token when file doesn't exist""" - with ( - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=False), - ): - mock_path.return_value = "/test/path/token.json" - - result = load_token() - - assert result is None - - def test_load_token_json_decode_error(self): - """Test loading token with invalid JSON""" - with ( - patch("builtins.open", mock_open(read_data="invalid json")), - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=True), - ): - mock_path.return_value = "/test/path/token.json" - - result = load_token() - - assert result is None - - def test_load_token_io_error(self): - """Test loading token with IO error""" - with ( - patch("builtins.open", side_effect=OSError("Permission denied")), - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=True), - ): - mock_path.return_value = "/test/path/token.json" - - result = load_token() - - assert result is None - - def test_clear_token_file_exists(self): - """Test clearing token when file exists""" - with ( - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=True), - patch("os.remove") as mock_remove, - ): - mock_path.return_value = "/test/path/token.json" - - clear_token() - - mock_remove.assert_called_once_with("/test/path/token.json") - - def test_clear_token_file_not_exists(self): - """Test clearing token when file doesn't exist""" - with ( - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=False), - patch("os.remove") as mock_remove, - ): - mock_path.return_value = "/test/path/token.json" - - clear_token() - - mock_remove.assert_not_called() - - def test_get_stored_api_key_success(self): - """Test getting stored API key successfully""" - token_data = {"key": "test-api-key-123", "user_id": "test-user"} - - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - result = get_stored_api_key() - assert result == "test-api-key-123" - - def test_get_stored_api_key_no_token(self): - """Test getting stored API key when no token exists""" - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=None, - ): - result = get_stored_api_key() - assert result is None - - def test_get_stored_api_key_no_key_field(self): - """Test getting stored API key when token has no key field""" - token_data = {"user_id": "test-user"} - - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - result = get_stored_api_key() - assert result is None - - def test_get_stored_api_key_base_url_match(self): - """Stored key is returned when expected_base_url matches stored origin""" - token_data = {"key": "sk-prod", "base_url": "https://real-proxy.com"} - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - assert get_stored_api_key(expected_base_url="https://real-proxy.com") == "sk-prod" - - def test_get_stored_api_key_base_url_match_trailing_slash(self): - """Trailing slash on expected_base_url is normalised before comparison""" - token_data = {"key": "sk-prod", "base_url": "https://real-proxy.com"} - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - assert get_stored_api_key(expected_base_url="https://real-proxy.com/") == "sk-prod" - - def test_get_stored_api_key_base_url_mismatch(self): - """Stored key is NOT returned when expected_base_url differs from stored origin""" - token_data = {"key": "sk-prod", "base_url": "https://real-proxy.com"} - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - assert get_stored_api_key(expected_base_url="https://evil.com") is None - - def test_get_stored_api_key_old_token_no_base_url(self): - """Old tokens without a base_url field are rejected when origin check is requested""" - token_data = {"key": "sk-old-token"} - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - assert get_stored_api_key(expected_base_url="https://real-proxy.com") is None + assert get_stored_api_key("https://real-proxy.com", vault=secret_vault_factory()) is None class TestLoginCommand: @@ -402,7 +269,7 @@ class TestLoginCommand: return_value=_mock_cli_sso_start_response(login_id="cli-test-uuid-123"), ) as mock_post, patch("requests.get", return_value=mock_response) as mock_get, - patch("litellm.proxy.client.cli.commands.auth.save_token") as mock_save, + patch("litellm.proxy.client.cli.commands.auth.save_cli_token") as mock_save, patch("litellm.proxy.client.cli.interface.show_commands") as mock_show_commands, ): result = self.runner.invoke(login, obj=mock_context.obj) @@ -424,8 +291,8 @@ class TestLoginCommand: # Verify JWT was saved mock_save.assert_called_once() saved_data = mock_save.call_args[0][0] - assert saved_data["key"] == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.jwt" - assert saved_data["user_id"] == "test-user-123" + assert saved_data.key == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.jwt" + assert saved_data.user_id == "test-user-123" # Verify commands were shown mock_show_commands.assert_called_once() @@ -557,7 +424,7 @@ class TestLogoutCommand: def test_logout_success(self): """Test successful logout""" - with patch("litellm.proxy.client.cli.commands.auth.clear_token") as mock_clear: + with patch("litellm.proxy.client.cli.commands.auth.clear_cli_token") as mock_clear: result = self.runner.invoke(logout) assert result.exit_code == 0 @@ -574,14 +441,15 @@ class TestWhoamiCommand: def test_whoami_authenticated(self): """Test whoami when user is authenticated""" - token_data = { - "user_email": "test@example.com", - "user_id": "test-user-123", - "user_role": "admin", - "timestamp": time.time() - 3600, # 1 hour ago - } + token_data = CliTokenRecord( + user_email="test@example.com", + user_id="test-user-123", + user_role="admin", + key="sk-live", + timestamp=time.time() - 3600, + ) - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=token_data): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=token_data): result = self.runner.invoke(whoami) assert result.exit_code == 0 @@ -593,7 +461,7 @@ class TestWhoamiCommand: def test_whoami_not_authenticated(self): """Test whoami when user is not authenticated""" - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=None): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=None): result = self.runner.invoke(whoami) assert result.exit_code == 0 @@ -602,14 +470,15 @@ class TestWhoamiCommand: def test_whoami_old_token(self): """Test whoami with old token showing warning""" - token_data = { - "user_email": "test@example.com", - "user_id": "test-user-123", - "user_role": "admin", - "timestamp": time.time() - (25 * 3600), # 25 hours ago - } + token_data = CliTokenRecord( + user_email="test@example.com", + user_id="test-user-123", + user_role="admin", + key="sk-live", + timestamp=time.time() - (25 * 3600), + ) - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=token_data): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=token_data): result = self.runner.invoke(whoami) assert result.exit_code == 0 @@ -618,12 +487,9 @@ class TestWhoamiCommand: def test_whoami_missing_fields(self): """Test whoami with token missing some fields""" - token_data = { - "timestamp": time.time() - 3600 - # Missing user_email, user_id, user_role - } + token_data = CliTokenRecord(key="sk-live", timestamp=time.time() - 3600) - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=token_data): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=token_data): result = self.runner.invoke(whoami) assert result.exit_code == 0 @@ -632,16 +498,16 @@ class TestWhoamiCommand: def test_whoami_no_timestamp(self): """Test whoami with token missing timestamp""" - token_data = { - "user_email": "test@example.com", - "user_id": "test-user-123", - "user_role": "admin", - # Missing timestamp - } + token_data = CliTokenRecord( + user_email="test@example.com", + user_id="test-user-123", + user_role="admin", + key="sk-live", + ) with ( patch( - "litellm.proxy.client.cli.commands.auth.load_token", + "litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=token_data, ), patch("time.time", return_value=1000), @@ -701,7 +567,7 @@ class TestCLIKeyRegenerationFlow: return_value=_mock_cli_sso_start_response(login_id="cli-session-uuid-456"), ), patch("requests.get", side_effect=[mock_first_response, mock_second_response]) as mock_get, - patch("litellm.proxy.client.cli.commands.auth.save_token") as mock_save, + patch("litellm.proxy.client.cli.commands.auth.save_cli_token") as mock_save, patch("litellm.proxy.client.cli.interface.show_commands") as mock_show_commands, patch("click.prompt", return_value="2"), ): # User selects index 2 @@ -734,8 +600,8 @@ class TestCLIKeyRegenerationFlow: # Verify JWT was saved mock_save.assert_called_once() saved_data = mock_save.call_args[0][0] - assert saved_data["key"] == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.team-beta.jwt" - assert saved_data["user_id"] == "test-user-456" + assert saved_data.key == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.team-beta.jwt" + assert saved_data.user_id == "test-user-456" mock_show_commands.assert_called_once() @@ -762,7 +628,7 @@ class TestCLIKeyRegenerationFlow: return_value=_mock_cli_sso_start_response(login_id="cli-session-uuid-solo"), ), patch("requests.get", return_value=mock_response), - patch("litellm.proxy.client.cli.commands.auth.save_token") as mock_save, + patch("litellm.proxy.client.cli.commands.auth.save_cli_token") as mock_save, patch("litellm.proxy.client.cli.interface.show_commands"), ): result = self.runner.invoke(login, obj=mock_context.obj) @@ -780,8 +646,8 @@ class TestCLIKeyRegenerationFlow: # Verify JWT was saved mock_save.assert_called_once() saved_data = mock_save.call_args[0][0] - assert saved_data["key"] == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.no-team.jwt" - assert saved_data["user_id"] == "test-user-solo" + assert saved_data.key == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.no-team.jwt" + assert saved_data.user_id == "test-user-solo" class TestPrintTokenCommand: @@ -810,7 +676,7 @@ class TestPrintTokenCommand: self.runner = CliRunner() def test_no_stored_token_fails_cleanly(self): - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=None): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=None): result = self.runner.invoke(print_token, obj={}) assert result.exit_code != 0 @@ -822,12 +688,12 @@ class TestPrintTokenCommand: one). Must use token.json's own base_url, not a hardcoded default.""" with ( patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "https://litellm-proxy.corp.com", - "key": "sk-prod-fresh", - "timestamp": time.time(), - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="https://litellm-proxy.corp.com", + key="sk-prod-fresh", + timestamp=time.time(), + ), ), patch("requests.post") as mock_post, ): @@ -844,12 +710,12 @@ class TestPrintTokenCommand: token minted for proxy A must not reach a helper invocation aimed at proxy B, even though the token itself is otherwise fresh.""" with patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "https://other-server.com", - "key": "sk-should-not-print", - "timestamp": time.time(), - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="https://other-server.com", + key="sk-should-not-print", + timestamp=time.time(), + ), ): result = self.runner.invoke( print_token, @@ -863,12 +729,12 @@ class TestPrintTokenCommand: """`lite up`'s own bound invocation shape: --base-url matching the token's origin must succeed exactly like the bare/legacy invocation does.""" with patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "http://localhost:4000", - "key": "sk-matches", - "timestamp": time.time(), - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="http://localhost:4000", + key="sk-matches", + timestamp=time.time(), + ), ): result = self.runner.invoke( print_token, @@ -884,12 +750,12 @@ class TestPrintTokenCommand: frequently).""" with ( patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "http://localhost:4000", - "key": "sk-cached-fresh", - "timestamp": time.time(), - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="http://localhost:4000", + key="sk-cached-fresh", + timestamp=time.time(), + ), ), patch("requests.post") as mock_post, ): @@ -908,12 +774,12 @@ class TestPrintTokenCommand: with ( patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "http://localhost:4000", - "key": "sk-stale-key", - "timestamp": old_timestamp, - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="http://localhost:4000", + key="sk-stale-key", + timestamp=old_timestamp, + ), ), patch("requests.post") as mock_post, ): @@ -925,25 +791,11 @@ class TestPrintTokenCommand: mock_post.assert_not_called() -def _write_home_json(home: Path, filename: str, payload: dict[str, object]) -> None: - litellm_dir = home / ".litellm" - litellm_dir.mkdir(exist_ok=True) - (litellm_dir / filename).write_text(json.dumps(payload)) - - class TestPrintTokenWithConfigFile: """A config-file base_url is a drop-in replacement for exporting LITELLM_PROXY_URL, so print-token must treat it as an explicit server choice: a token minted for a different proxy is never handed out.""" - @pytest.fixture - def isolated_home(self, monkeypatch, tmp_path): - monkeypatch.setenv("HOME", str(tmp_path)) - monkeypatch.setenv("USERPROFILE", str(tmp_path)) - monkeypatch.delenv("LITELLM_PROXY_URL", raising=False) - monkeypatch.delenv("LITELLM_PROXY_API_KEY", raising=False) - return tmp_path - def test_config_base_url_mismatch_fails_closed(self, isolated_home): _write_home_json( isolated_home, @@ -1001,37 +853,196 @@ class TestPrintTokenWithConfigFile: assert result.stdout.strip() == "sk-issued-for-a" -class TestSaveTokenPrivateWrite: - """token.json holds the real API key: it must never be world-readable at any - instant, and a failed write must not destroy the previously stored token.""" +class TestFileFallbackStorage: + """On a headless box with no keychain the token file is still the only store, so it has to + stay owner-only and survive a failed write.""" - @pytest.fixture - def isolated_home(self, monkeypatch, tmp_path): - monkeypatch.setenv("HOME", str(tmp_path)) - monkeypatch.setenv("USERPROFILE", str(tmp_path)) - monkeypatch.delenv("LITELLM_PROXY_URL", raising=False) - monkeypatch.delenv("LITELLM_PROXY_API_KEY", raising=False) - return tmp_path - - def test_save_token_owner_only_permissions_and_no_temp_leftovers(self, isolated_home): - save_token({"key": "sk-secret", "user_id": "u-1", "timestamp": 1234567890}) + def test_owner_only_file_and_directory_with_no_temp_leftovers(self, isolated_home, secret_vault_factory): + save_cli_token( + CliTokenRecord(base_url="https://proxy.example.com", key="sk-secret", user_id="u-1", timestamp=1234567890), + vault=secret_vault_factory(available=False), + ) token_file = isolated_home / ".litellm" / "token.json" - assert json.loads(token_file.read_text()) == {"key": "sk-secret", "user_id": "u-1", "timestamp": 1234567890} + assert json.loads(token_file.read_text())["key"] == "sk-secret" assert stat.S_IMODE(token_file.stat().st_mode) == 0o600 + assert stat.S_IMODE(token_file.parent.stat().st_mode) == 0o700 assert list(token_file.parent.glob(".tmp-*")) == [] - def test_save_token_failure_mid_write_preserves_existing_token(self, isolated_home): + def test_a_failed_write_preserves_the_existing_token(self, isolated_home, secret_vault_factory, monkeypatch): _write_home_json(isolated_home, "token.json", {"key": "sk-original", "timestamp": 1234567890}) token_file = isolated_home / ".litellm" / "token.json" + def _explode(*args, **kwargs): + raise TypeError("not serialisable") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + with pytest.raises(TypeError): - save_token({"key": object()}) + save_cli_token(CliTokenRecord(key="sk-new"), vault=secret_vault_factory(available=False)) assert json.loads(token_file.read_text()) == {"key": "sk-original", "timestamp": 1234567890} assert list(token_file.parent.glob(".tmp-*")) == [] +class TestKeychainBackedCommands: + """End-to-end through the `lite` commands: the secret lives in the keychain, the file keeps + only metadata, and every command still reads and writes through that split.""" + + def setup_method(self): + self.runner = CliRunner() + + def _login(self, vault, base_url="https://test.example.com"): + poll_response = Mock() + poll_response.status_code = 200 + poll_response.json.return_value = { + "status": "ready", + "key": "sk-minted", + "user_id": "test-user-123", + "team_id": "team-1", + "teams": ["team-1"], + } + with ( + patch("webbrowser.open"), + patch("requests.post", return_value=_mock_cli_sso_start_response()), + patch("requests.get", return_value=poll_response), + patch("litellm.proxy.client.cli.interface.show_commands"), + ): + return self.runner.invoke(login, obj={"base_url": base_url, "secret_vault": vault}) + + def test_login_puts_the_secret_in_the_keychain_and_not_in_the_file(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + + result = self._login(vault) + + token_file = isolated_home / ".litellm" / "token.json" + assert result.exit_code == 0 + assert "Credential stored in your OS keychain." in result.output + assert json.loads(vault.blob)["key"] == "sk-minted" + assert "sk-minted" not in token_file.read_text() + assert json.loads(token_file.read_text())["user_id"] == "test-user-123" + + def test_login_without_a_keychain_says_where_the_credential_went(self, isolated_home, secret_vault_factory): + result = self._login(secret_vault_factory(available=False)) + + token_file = isolated_home / ".litellm" / "token.json" + assert result.exit_code == 0 + assert "No OS keychain available" in result.output + assert str(token_file) in result.output + assert json.loads(token_file.read_text())["key"] == "sk-minted" + + def test_whoami_and_print_token_read_through_the_keychain(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + self._login(vault) + obj = {"base_url": "https://test.example.com", "secret_vault": vault} + + whoami_result = self.runner.invoke(whoami, obj=obj) + print_result = self.runner.invoke(print_token, obj=obj) + + assert "Authenticated" in whoami_result.output + assert "test-user-123" in whoami_result.output + assert print_result.exit_code == 0 + assert print_result.stdout.strip() == "sk-minted" + + def test_logout_clears_the_keychain_as_well_as_the_file(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + self._login(vault) + + result = self.runner.invoke(logout, obj={"base_url": "https://test.example.com", "secret_vault": vault}) + + assert result.exit_code == 0 + assert "Logged out successfully" in result.output + assert vault.blob is None + assert not (isolated_home / ".litellm" / "token.json").exists() + + def test_logout_warns_when_the_keychain_will_not_release_the_secret(self, isolated_home, secret_vault_factory): + """Silently reporting success would leave a live credential in the keychain.""" + vault = secret_vault_factory(erasable=False) + self._login(vault) + + result = self.runner.invoke(logout, obj={"base_url": "https://test.example.com", "secret_vault": vault}) + + assert result.exit_code == 0 + assert "could not be removed" in result.output + assert not (isolated_home / ".litellm" / "token.json").exists() + + def test_print_token_explains_a_locked_keychain_instead_of_printing_nothing( + self, isolated_home, secret_vault_factory + ): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "https://test.example.com", "user_id": "u-1", "timestamp": time.time()}, + ) + obj = {"base_url": "https://test.example.com", "secret_vault": secret_vault_factory(available=False)} + + result = self.runner.invoke(print_token, obj=obj) + + assert result.exit_code == 1 + assert KEYCHAIN_UNREACHABLE_MESSAGE in result.output + + def test_whoami_flags_a_locked_keychain(self, isolated_home, secret_vault_factory): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "https://test.example.com", "user_id": "u-1", "timestamp": time.time()}, + ) + obj = {"base_url": "https://test.example.com", "secret_vault": secret_vault_factory(available=False)} + + result = self.runner.invoke(whoami, obj=obj) + + assert "Authenticated" in result.output + assert KEYCHAIN_UNREACHABLE_MESSAGE in result.output + + +class TestApiKeyPrecedence: + """`LITELLM_PROXY_API_KEY` and `--api-key` outrank the stored credential; moving the secret + into the keychain must not disturb that order.""" + + def _resolved_key(self, args, obj=None): + with patch("litellm.proxy.client.cli.main.print_version") as mock_print_version: + result = CliRunner().invoke(cli, [*args, "version"], obj=obj) + assert result.exit_code == 0, result.output + return mock_print_version.call_args[0][1] + + def test_the_stored_credential_is_the_fallback(self, isolated_home): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "http://localhost:4000", "key": "sk-stored", "timestamp": time.time()}, + ) + + assert self._resolved_key([]) == "sk-stored" + + def test_the_stored_credential_is_read_through_the_injected_keychain(self, isolated_home, secret_vault_factory): + """The vault handed to the CLI through ctx.obj must be the one the group callback reads, + so a keychain-held secret resolves without ever touching the host OS keychain.""" + _write_home_json(isolated_home, "token.json", {"base_url": "http://localhost:4000", "timestamp": time.time()}) + vault = secret_vault_factory(_secret_blob("http://localhost:4000", "sk-keychain")) + + assert self._resolved_key([], obj={"secret_vault": vault}) == "sk-keychain" + + def test_env_var_beats_the_stored_credential(self, isolated_home, monkeypatch): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "http://localhost:4000", "key": "sk-stored", "timestamp": time.time()}, + ) + monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-from-env") + + assert self._resolved_key([]) == "sk-from-env" + + def test_explicit_api_key_beats_both(self, isolated_home, monkeypatch): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "http://localhost:4000", "key": "sk-stored", "timestamp": time.time()}, + ) + monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-from-env") + + assert self._resolved_key(["--api-key", "sk-explicit"]) == "sk-explicit" + + class TestLoginConfigClaude: """`lite login --config-claude` wiring into ~/.claude/settings.json""" @@ -1054,7 +1065,7 @@ class TestLoginConfigClaude: patch("webbrowser.open"), patch("requests.post", return_value=_mock_cli_sso_start_response()), patch("requests.get", return_value=poll_response), - patch("litellm.proxy.client.cli.commands.auth.save_token"), + patch("litellm.proxy.client.cli.commands.auth.save_cli_token"), patch("litellm.proxy.client.cli.interface.show_commands"), patch("litellm.proxy.client.cli.commands.auth.CLAUDE_SETTINGS_PATH", settings_path), patch( diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py index bc9744eb410..9010fb4c022 100644 --- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -7,6 +7,7 @@ from unittest.mock import patch import pytest from click.testing import CliRunner +from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.claude_settings import ( AUTOROUTE_BACKUP_PATH, @@ -181,18 +182,18 @@ class TestApiKeyHelperIsActuallyInvocable: assert result.exit_code != 2 def test_the_generated_command_reaches_print_token(self): - with patch(f"{AUTH_MODULE}.load_token", return_value=None): + with patch(f"{AUTH_MODULE}.load_cli_token", return_value=None): result = CliRunner().invoke(cli, self._helper_args("http://localhost:4000")) assert "Not authenticated" in result.output def test_the_generated_command_carries_the_base_url_through(self): - stale = { - "base_url": "http://other-proxy.example.com", - "key": "sk-stale", - "timestamp": time.time(), - } - with patch(f"{AUTH_MODULE}.load_token", return_value=stale): + stale = CliTokenRecord( + base_url="http://other-proxy.example.com", + key="sk-stale", + timestamp=time.time(), + ) + with patch(f"{AUTH_MODULE}.load_cli_token", return_value=stale): result = CliRunner().invoke(cli, self._helper_args("http://localhost:4000")) assert "Not authenticated for this server" in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_config_commands.py b/tests/test_litellm/proxy/client/cli/test_config_commands.py index d81ee6bd2b1..6f3f4e4b268 100644 --- a/tests/test_litellm/proxy/client/cli/test_config_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_config_commands.py @@ -18,7 +18,7 @@ from litellm.proxy.client.cli.commands.config import ( load_config, save_config, ) -from litellm.proxy.client.cli.commands.private_json import write_private_json +from litellm.litellm_core_utils.private_json import write_private_json from litellm.proxy.client.cli.interface import show_commands @@ -355,7 +355,7 @@ class TestWritePrivateJson: def _interrupt(*args: object, **kwargs: object) -> None: raise KeyboardInterrupt() - monkeypatch.setattr("litellm.proxy.client.cli.commands.private_json.json.dump", _interrupt) + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _interrupt) target = tmp_path / "config.json" with pytest.raises(KeyboardInterrupt): diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index 51de0dcf11d..aebf441f777 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -8,6 +8,7 @@ import click import pytest from click.testing import CliRunner +from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord from litellm.proxy.client.cli.commands import up as up_module from litellm.proxy.client.cli.commands.agents import AgentRunError from litellm.proxy.client.cli.commands.claude_settings import ClaudeSettingsError @@ -220,13 +221,17 @@ def _make_ctx(base_url): return click.Context(click.Command("test"), obj={"base_url": base_url}) +def _token(key, base_url): + return CliTokenRecord(key=key, base_url=base_url) + + class TestEnsureFreshLogin: """A token that is fresh but was issued for a *different* proxy must not be trusted: without this check, a user logged into proxy A who runs `up --base-url proxy-b` would silently get an apiKeyHelper wired up around proxy A's real token, which print-token would then hand to proxy B.""" def test_reuses_a_fresh_token_issued_for_the_same_proxy(self, monkeypatch): - monkeypatch.setattr(up_module, "load_token", lambda: {"key": "sk-a", "base_url": "http://proxy-a:4000"}) + monkeypatch.setattr(up_module, "load_cli_token", lambda **_: _token("sk-a", "http://proxy-a:4000")) monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) login_calls = [] monkeypatch.setattr(up_module, "login", lambda ctx: login_calls.append(ctx)) @@ -239,11 +244,11 @@ class TestEnsureFreshLogin: monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: True) tokens = iter( [ - {"key": "sk-a", "base_url": "http://proxy-a:4000"}, - {"key": "sk-b", "base_url": "http://proxy-b:4000"}, + _token("sk-a", "http://proxy-a:4000"), + _token("sk-b", "http://proxy-b:4000"), ] ) - monkeypatch.setattr(up_module, "load_token", lambda: next(tokens)) + monkeypatch.setattr(up_module, "load_cli_token", lambda **_: next(tokens)) monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) login_calls = [] @@ -259,7 +264,7 @@ class TestEnsureFreshLogin: def test_fails_cleanly_non_interactively_when_only_a_different_proxys_token_is_cached(self, monkeypatch): monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: False) - monkeypatch.setattr(up_module, "load_token", lambda: {"key": "sk-a", "base_url": "http://proxy-a:4000"}) + monkeypatch.setattr(up_module, "load_cli_token", lambda **_: _token("sk-a", "http://proxy-a:4000")) monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) with pytest.raises(UpError, match="lite login"): @@ -276,7 +281,7 @@ class TestUpCommand: backup_path.write_text(json.dumps(existing_backup)) with ( - patch(f"{UP_MODULE}.load_token", return_value={"key": "sk-fresh", "base_url": "http://localhost:4000"}), + patch(f"{UP_MODULE}.load_cli_token", return_value=_token("sk-fresh", "http://localhost:4000")), patch(f"{UP_MODULE}.is_cli_token_fresh", return_value=True), patch(f"{UP_MODULE}.resolve_api_key", return_value="sk-fresh"), patch(f"{UP_MODULE}.verify_proxy_key"), @@ -293,7 +298,7 @@ class TestUpCommand: _patch_paths(monkeypatch, tmp_path) monkeypatch.setattr(sys.stdin, "isatty", lambda: False) - with patch(f"{UP_MODULE}.load_token", return_value=None): + with patch(f"{UP_MODULE}.load_cli_token", return_value=None): result = self.runner.invoke(up, obj={"base_url": "http://localhost:4000"}) assert result.exit_code != 0 @@ -303,7 +308,7 @@ class TestUpCommand: _patch_paths(monkeypatch, tmp_path) with ( - patch(f"{UP_MODULE}.load_token", return_value={"key": "sk-fresh", "base_url": "http://localhost:4000"}), + patch(f"{UP_MODULE}.load_cli_token", return_value=_token("sk-fresh", "http://localhost:4000")), patch(f"{UP_MODULE}.is_cli_token_fresh", return_value=True), patch(f"{UP_MODULE}.resolve_api_key", return_value="sk-fresh"), patch( @@ -329,7 +334,7 @@ class TestUpCommand: return True with ( - patch(f"{UP_MODULE}.load_token", return_value={"key": "sk-fresh", "base_url": "http://localhost:4000"}), + patch(f"{UP_MODULE}.load_cli_token", return_value=_token("sk-fresh", "http://localhost:4000")), patch(f"{UP_MODULE}.is_cli_token_fresh", return_value=True), patch(f"{UP_MODULE}.resolve_api_key", return_value="sk-fresh"), patch(f"{UP_MODULE}.verify_proxy_key"), diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 31726acfbaa..a5c5a9f135b 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22806 + "limit": 22805 }, "LIT002": { - "limit": 26878 + "limit": 26877 }, "LIT003": { "limit": 269 @@ -27,12 +27,12 @@ "limit": 0 }, "LIT010": { - "limit": 16695 + "limit": 16693 }, "LIT011": { "limit": 5588 }, "LIT012": { - "limit": 4519 + "limit": 4511 } } diff --git a/uv.lock b/uv.lock index d9e2fb94667..53bb0cb8f82 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-16T00:41:08.185444Z" +exclude-newer = "2026-08-17T01:06:38.502388Z" exclude-newer-span = "P3D" [manifest] @@ -710,6 +710,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, ] +[[package]] +name = "backports-tarfile" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", size = 86406, upload-time = "2024-05-28T17:01:54.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, +] + [[package]] name = "basedpyright" version = "1.39.7" @@ -3538,6 +3547,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, ] +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-tarfile", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, +] + +[[package]] +name = "jaraco-functools" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/1f/c23395957d41ccf27c4e535c3d334c4051e5395b3752057ba4cbaec35c56/jaraco_functools-4.6.0.tar.gz", hash = "sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280", size = 20837, upload-time = "2026-07-14T01:28:02.544Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/36/ecc85bc96c273dc8a11273ed4782272975e6338d4a3e9228621175edf0e3/jaraco_functools-4.6.0-py3-none-any.whl", hash = "sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30", size = 11677, upload-time = "2026-07-14T01:28:01.59Z" }, +] + +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -3764,6 +3818,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "keyring" +version = "25.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata", marker = "python_full_version < '3.12'" }, + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, +] + [[package]] name = "kiwisolver" version = "1.5.0" @@ -4222,6 +4294,7 @@ caching = [ ] cli = [ { name = "inquirerpy" }, + { name = "keyring" }, { name = "pyyaml" }, { name = "requests" }, { name = "rich" }, @@ -4352,6 +4425,7 @@ dev = [ { name = "diff-cover" }, { name = "fakeredis" }, { name = "fastapi-offline" }, + { name = "keyring" }, { name = "langfuse" }, { name = "openapi-core" }, { name = "opentelemetry-api" }, @@ -4446,6 +4520,7 @@ requires-dist = [ { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" }, { name = "jinja2", specifier = ">=3.1.6,<4.0" }, { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, + { name = "keyring", marker = "extra == 'cli'", specifier = ">=25.6.0,<26.0" }, { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = ">=2.59.7,<3.0" }, { name = "litellm-enterprise", marker = "extra == 'proxy'", editable = "enterprise" }, { name = "litellm-proxy-extras", marker = "extra == 'proxy'", editable = "litellm-proxy-extras" }, @@ -4532,6 +4607,7 @@ dev = [ { name = "diff-cover", specifier = "==9.7.2" }, { name = "fakeredis", specifier = "==2.34.1" }, { name = "fastapi-offline", specifier = "==1.7.6" }, + { name = "keyring", specifier = "==25.7.0" }, { name = "langfuse", specifier = "==2.59.7" }, { name = "openapi-core", specifier = "==0.22.0" }, { name = "opentelemetry-api", specifier = "==1.28.0" }, @@ -7790,6 +7866,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" }, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -8632,6 +8717,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, ] +[[package]] +name = "secretstorage" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jeepney" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, +] + [[package]] name = "semantic-router" version = "0.1.15" From bd322ed8a7eb6968b2af8bebce28eb9f19251fd3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:11:01 -0700 Subject: [PATCH 04/53] refactor(cli): state the credential-store precedence rules as contracts Drop the inline notes on keychain erasure and disk-vs-vault precedence in favour of docstrings on the two functions that own those rules, and remove a stale section header and a field note that the code already says plainly. --- litellm/litellm_core_utils/cli_keyring.py | 6 +++++- litellm/litellm_core_utils/cli_token_utils.py | 6 +++++- litellm/proxy/client/cli/commands/auth.py | 3 --- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index 873db64a728..fcbf5ada55a 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -98,10 +98,14 @@ class KeyringVault: return True def erase(self) -> bool: + """Whether the keychain is guaranteed to hold no credential afterwards. + + An uninstalled `keyring` package can never have stored one. A kill switch set after + a credential was stored leaves that entry out of reach, so erasure cannot be promised. + """ if _import_keyring() is None: return True if _keyring_disabled(): - # a credential stored before the kill switch was set may still be in the keychain return False match self.read(): case SecretUnavailable(): diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 9960192180c..dd263ccd412 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -168,8 +168,12 @@ def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecor def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) -> CliTokenRecord | None: + """Resolve the credential when both stores hold one. + + A secret still on disk is the fresher of the two, because it is only left there when the + keychain write that should have removed it failed, so it outranks the vault entry. + """ if record.key is not None: - # a secret still on disk means the last keychain write failed: the file outranks the vault return _migrate_file_secret(record, vault) try: secret: Final = CliTokenSecret.model_validate_json(blob) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 8b9ef5633da..c2b5b6a620f 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -76,7 +76,6 @@ KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( ) -# Token storage utilities def context_secret_vault(ctx: click.Context) -> SecretVault: """Where this invocation reads and writes secret material; injectable through ctx.obj for tests""" ctx_obj: Final[CliContextObj | None] = ctx.obj @@ -666,8 +665,6 @@ def login(ctx: click.Context, config_claude: bool): api_key: Final = auth_result["api_key"] user_id: Final = auth_result["user_id"] - # base_url is stored so we can verify origin before reusing the - # key on a subsequent CLI invocation. record: Final = CliTokenRecord( base_url=base_url.rstrip("/"), key=api_key, From 01add582982a253c2fff3466b608c1d0409ed1ae Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:45:38 -0700 Subject: [PATCH 05/53] fix(cli): name why a login fell back to the token file lite ships with every install of litellm, but the keyring package it needs for keychain storage only ships with the cli extra. Such a user on a Mac was told 'No OS keychain available' about a machine that plainly has one, with nothing pointing at the missing package. The vault now reports which of the three unusable states it is in, so login can point at the install, name the kill switch, or report a genuinely absent keychain. --- litellm/litellm_core_utils/cli_keyring.py | 56 +++++++++++++------ litellm/litellm_core_utils/cli_token_utils.py | 26 +++++---- litellm/proxy/client/README.md | 2 +- litellm/proxy/client/cli/commands/auth.py | 41 +++++++++++--- tests/test_litellm/conftest.py | 18 ++++-- .../test_cli_token_utils.py | 23 ++++---- .../proxy/client/cli/test_auth_commands.py | 28 ++++++++++ 7 files changed, 141 insertions(+), 53 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index fcbf5ada55a..b19b3d3bc83 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -5,9 +5,9 @@ SDK-level access to the OS keychain (macOS Keychain, Windows Credential Manager, Linux Secret Service) that holds the credential minted by `lite login`. The `keyring` package is optional and imported lazily, so importing this module -never pulls it in. Every failure is returned as a value: a machine with no -keychain, or one whose keychain is locked, must degrade to the token file rather -than break `lite` or the SDK. +never pulls it in. Every failure is returned as a value, naming which of the +three ways the keychain can be out of reach applies, so callers can degrade to +the token file and tell the user what to do about it. """ import os @@ -32,11 +32,28 @@ class SecretMissing: @dataclass(frozen=True, slots=True) -class SecretUnavailable: +class SecretStored: pass -SecretRead: TypeAlias = SecretFound | SecretMissing | SecretUnavailable +@dataclass(frozen=True, slots=True) +class KeyringNotInstalled: + pass + + +@dataclass(frozen=True, slots=True) +class KeyringDisabled: + pass + + +@dataclass(frozen=True, slots=True) +class KeyringUnreachable: + pass + + +KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable +SecretRead: TypeAlias = SecretFound | SecretMissing | KeyringUnusable +SecretWrite: TypeAlias = SecretStored | KeyringUnusable class SecretVault(Protocol): @@ -44,7 +61,7 @@ class SecretVault(Protocol): def read(self) -> SecretRead: ... - def write(self, blob: str) -> bool: ... + def write(self, blob: str) -> SecretWrite: ... def erase(self) -> bool: ... @@ -69,8 +86,11 @@ def _import_keyring() -> KeyringApi | None: return keyring -def _keyring_api() -> KeyringApi | None: - return None if _keyring_disabled() else _import_keyring() +def _keyring_api() -> KeyringApi | KeyringNotInstalled | KeyringDisabled: + if _keyring_disabled(): + return KeyringDisabled() + api: Final = _import_keyring() + return KeyringNotInstalled() if api is None else api @dataclass(frozen=True, slots=True) @@ -79,23 +99,23 @@ class KeyringVault: def read(self) -> SecretRead: api: Final = _keyring_api() - if api is None: - return SecretUnavailable() + if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): + return api try: blob: Final = api.get_password(KEYRING_SERVICE, KEYRING_ACCOUNT) except Exception: # noqa: BLE001 # backends raise outside keyring.errors; never break the SDK - return SecretUnavailable() + return KeyringUnreachable() return SecretMissing() if blob is None else SecretFound(blob) - def write(self, blob: str) -> bool: + def write(self, blob: str) -> SecretWrite: api: Final = _keyring_api() - if api is None: - return False + if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): + return api try: api.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, blob) except Exception: # noqa: BLE001 # a keychain that refuses the write falls back to the token file - return False - return True + return KeyringUnreachable() + return SecretStored() def erase(self) -> bool: """Whether the keychain is guaranteed to hold no credential afterwards. @@ -108,7 +128,7 @@ class KeyringVault: if _keyring_disabled(): return False match self.read(): - case SecretUnavailable(): + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): return False case SecretMissing(): return True @@ -117,7 +137,7 @@ class KeyringVault: def _delete(self) -> bool: api: Final = _keyring_api() - if api is None: + if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): return False try: api.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index dd263ccd412..40822e5b335 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -22,10 +22,14 @@ from pydantic import BaseModel, ConfigDict, ValidationError from litellm.litellm_core_utils.cli_keyring import ( SYSTEM_KEYRING, + KeyringDisabled, + KeyringNotInstalled, + KeyringUnreachable, SecretFound, SecretMissing, - SecretUnavailable, + SecretStored, SecretVault, + SecretWrite, ) from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json @@ -79,13 +83,15 @@ def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | N return _resolve_secret(record, vault) -def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> bool: - """Store a freshly minted credential. Returns whether the keychain took the secret""" - if record.key is None or not vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)): - _write_token_file(record) - return False - _write_token_file(_without_secret(record)) - return True +def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretWrite: + """Store a freshly minted credential. Reports whether the keychain took the secret, and why not""" + outcome: Final = ( + SecretStored() + if record.key is None + else vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)) + ) + _write_token_file(_without_secret(record) if isinstance(outcome, SecretStored) else record) + return outcome def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> bool: @@ -163,7 +169,7 @@ def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecor return _apply_vault_secret(record, blob, vault) case SecretMissing(): return _migrate_file_secret(record, vault) - case SecretUnavailable(): + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): return record @@ -188,7 +194,7 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) - def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: if record.key is None: return None - if vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)): + if isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): _scrub_file_secret(record) return record diff --git a/litellm/proxy/client/README.md b/litellm/proxy/client/README.md index 9ece4c2be3d..d46c7174b4c 100644 --- a/litellm/proxy/client/README.md +++ b/litellm/proxy/client/README.md @@ -377,7 +377,7 @@ The key itself goes into the OS keychain (macOS Keychain, Windows Credential Man } ``` -Headless boxes and CI runners usually have no keychain. There the key stays in the same `0600` file alongside the metadata, exactly as it did before, and `lite login` tells you which of the two happened. Set `LITELLM_CLI_DISABLE_KEYRING=1` to force the file even where a keychain exists. A `token.json` written by an older `lite` keeps working and is moved into the keychain, and scrubbed from the file, the first time a keychain-capable `lite` reads it. +Keychain storage needs the `keyring` package, which ships with `pip install 'litellm[cli]'`. Headless boxes and CI runners usually have no keychain either. In all of those cases the key stays in the same `0600` file alongside the metadata, exactly as it did before, and `lite login` names which one applies: the package is missing, the machine has no keychain, or you set `LITELLM_CLI_DISABLE_KEYRING=1` to force the file even where a keychain exists. A `token.json` written by an older `lite` keeps working and is moved into the keychain, and scrubbed from the file, the first time a keychain-capable `lite` reads it. `lite logout` clears both stores. If the keychain is locked at that moment it says so, and re-running it once the keychain is unlocked finishes the job. diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index c2b5b6a620f..03906f9b6df 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -11,7 +11,16 @@ from rich.table import Table from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import CLI_JWT_EXPIRATION_HOURS -from litellm.litellm_core_utils.cli_keyring import SYSTEM_KEYRING, SecretVault +from litellm.litellm_core_utils.cli_keyring import ( + DISABLE_KEYRING_ENV_VAR, + SYSTEM_KEYRING, + KeyringDisabled, + KeyringNotInstalled, + KeyringUnreachable, + SecretStored, + SecretVault, + SecretWrite, +) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, clear_cli_token, @@ -72,9 +81,29 @@ class CliAuthResult(TypedDict): KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( - "Your credential is stored in your OS keychain, which could not be read. Unlock it and retry, or run 'lite login'." + "Your credential is stored in your OS keychain, which could not be read. Unlock it and retry, " + "install the keyring package with: pip install 'litellm[cli]', or run 'lite login'." ) +KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'" + + +def storage_notice(outcome: SecretWrite) -> str: + """Tell the user where the credential ended up, and how to get keychain storage if it did not.""" + path: Final = get_cli_token_file_path() + match outcome: + case SecretStored(): + return "Credential stored in your OS keychain." + case KeyringNotInstalled(): + return ( + f"Credential stored in {path} (owner-only). " + f"For OS keychain storage, install the keyring package with: {KEYRING_INSTALL_HINT}" + ) + case KeyringDisabled(): + return f"Keychain storage is off ({DISABLE_KEYRING_ENV_VAR}). Credential stored in {path} (owner-only)." + case KeyringUnreachable(): + return f"No OS keychain available. Credential stored in {path} (owner-only)." + def context_secret_vault(ctx: click.Context) -> SecretVault: """Where this invocation reads and writes secret material; injectable through ctx.obj for tests""" @@ -675,15 +704,11 @@ def login(ctx: click.Context, config_claude: bool): jwt_token="", timestamp=time.time(), ) - in_keychain: Final = save_cli_token(record, vault=context_secret_vault(ctx)) + stored: Final = save_cli_token(record, vault=context_secret_vault(ctx)) click.echo("\nLogin successful!") click.echo(f"JWT Token: {api_key[:20]}...") - click.echo( - "Credential stored in your OS keychain." - if in_keychain - else f"No OS keychain available; credential stored in {get_cli_token_file_path()} (owner-only)." - ) + click.echo(storage_notice(stored)) click.echo("You can now use the CLI without specifying --api-key") if config_claude: diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index ce0fd197538..a716ec0e0aa 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -23,10 +23,13 @@ from litellm import router as litellm_router_module from litellm import utils as litellm_utils_module from litellm._logging import ALL_LOGGERS from litellm.litellm_core_utils.cli_keyring import ( + KeyringUnreachable, + KeyringUnusable, SecretFound, SecretMissing, SecretRead, - SecretUnavailable, + SecretStored, + SecretWrite, ) from litellm.litellm_core_utils.prompt_templates import ( image_handling as image_handling_module, @@ -125,7 +128,8 @@ class FakeSecretVault: """In-memory stand-in for the OS keychain, injected wherever CLI credential storage is exercised. `available=False` models a keychain that is locked or has no backend, `writable=False` one that - refuses to store, and `erasable=False` one that will not release what it already holds. + refuses to store, `erasable=False` one that will not release what it already holds, and `failure` + picks which unusable state those report. """ def __init__( @@ -135,11 +139,13 @@ class FakeSecretVault: available: bool = True, writable: bool = True, erasable: bool = True, + failure: KeyringUnusable = KeyringUnreachable(), ) -> None: self.blob: str | None = blob self.available: bool = available self.writable: bool = writable self.erasable: bool = erasable + self.failure: KeyringUnusable = failure self.reads: int = 0 self.writes: list[str] = [] self.erases: int = 0 @@ -147,15 +153,15 @@ class FakeSecretVault: def read(self) -> SecretRead: self.reads += 1 if not self.available: - return SecretUnavailable() + return self.failure return SecretMissing() if self.blob is None else SecretFound(self.blob) - def write(self, blob: str) -> bool: + def write(self, blob: str) -> SecretWrite: self.writes.append(blob) if not (self.available and self.writable): - return False + return self.failure self.blob = blob - return True + return SecretStored() def erase(self) -> bool: self.erases += 1 diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 56ab6bcbfe0..961d986e8f5 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -13,7 +13,10 @@ from litellm.litellm_core_utils.cli_keyring import ( KeyringVault, SecretFound, SecretMissing, - SecretUnavailable, + KeyringDisabled, + KeyringNotInstalled, + KeyringUnreachable, + SecretStored, ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, @@ -253,7 +256,7 @@ class TestSaveCliToken: vault=vault, ) - assert stored is True + assert stored == SecretStored() assert "sk-new" not in _token_file(isolated_home).read_text() assert json.loads(vault.blob)["key"] == "sk-new" assert load_cli_token(vault=vault).key == "sk-new" @@ -265,7 +268,7 @@ class TestSaveCliToken: ) path = _token_file(isolated_home) - assert stored is False + assert stored == KeyringUnreachable() assert json.loads(path.read_text())["key"] == "sk-new" assert stat.S_IMODE(path.stat().st_mode) == 0o600 assert list(path.parent.glob(".tmp-*")) == [] @@ -379,7 +382,7 @@ class TestKeyringVault: fake = install_fake_keyring(_FakeKeyringModule()) vault = KeyringVault() - assert vault.write("blob-1") is True + assert vault.write("blob-1") == SecretStored() assert vault.read() == SecretFound("blob-1") assert vault.erase() is True assert vault.read() == SecretMissing() @@ -393,8 +396,8 @@ class TestKeyringVault: monkeypatch.setenv(DISABLE_KEYRING_ENV_VAR, "1") vault = KeyringVault() - assert vault.read() == SecretUnavailable() - assert vault.write("blob-1") is False + assert vault.read() == KeyringDisabled() + assert vault.write("blob-1") == KeyringDisabled() assert vault.erase() is False def test_an_uninstalled_keyring_library_degrades_to_the_file(self, monkeypatch): @@ -404,19 +407,19 @@ class TestKeyringVault: monkeypatch.setitem(sys.modules, "keyring", None) vault = KeyringVault() - assert vault.read() == SecretUnavailable() - assert vault.write("blob-1") is False + assert vault.read() == KeyringNotInstalled() + assert vault.write("blob-1") == KeyringNotInstalled() assert vault.erase() is True def test_a_locked_keychain_is_reported_not_raised(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("keyring is locked"))) - assert KeyringVault().read() == SecretUnavailable() + assert KeyringVault().read() == KeyringUnreachable() def test_a_refused_write_is_reported_not_raised(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(set_error=RuntimeError("no backend"))) - assert KeyringVault().write("blob-1") is False + assert KeyringVault().write("blob-1") == KeyringUnreachable() def test_a_refused_delete_is_reported_so_logout_can_warn(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(stored="blob-1", delete_error=RuntimeError("locked"))) diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index e93f05cb4aa..1f2d48f6547 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -13,6 +13,11 @@ import pytest from click.testing import CliRunner from litellm.constants import CLI_JWT_EXPIRATION_HOURS +from litellm.litellm_core_utils.cli_keyring import ( + DISABLE_KEYRING_ENV_VAR, + KeyringDisabled, + KeyringNotInstalled, +) from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord, save_cli_token from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.auth import ( @@ -931,6 +936,29 @@ class TestKeychainBackedCommands: assert str(token_file) in result.output assert json.loads(token_file.read_text())["key"] == "sk-minted" + def test_login_points_a_user_missing_the_keyring_package_at_the_install( + self, isolated_home, secret_vault_factory + ): + """`lite` ships with every install, the keyring package only with the cli extra. Telling + that user their machine has no keychain sends them looking for a problem they do not have.""" + result = self._login(secret_vault_factory(available=False, failure=KeyringNotInstalled())) + + token_file = isolated_home / ".litellm" / "token.json" + assert result.exit_code == 0 + assert "pip install 'litellm[cli]'" in result.output + assert "No OS keychain available" not in result.output + assert json.loads(token_file.read_text())["key"] == "sk-minted" + + def test_login_names_the_kill_switch_instead_of_blaming_the_machine( + self, isolated_home, secret_vault_factory + ): + result = self._login(secret_vault_factory(available=False, failure=KeyringDisabled())) + + assert result.exit_code == 0 + assert DISABLE_KEYRING_ENV_VAR in result.output + assert "No OS keychain available" not in result.output + assert json.loads((isolated_home / ".litellm" / "token.json").read_text())["key"] == "sk-minted" + def test_whoami_and_print_token_read_through_the_keychain(self, isolated_home, secret_vault_factory): vault = secret_vault_factory() self._login(vault) From 1750893a6905f45d7fa9cc65167856ae6c182208 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:26:41 -0700 Subject: [PATCH 06/53] fix(cli): never report success while a credential is still readable Migration moved the secret into the keychain and then suppressed any OSError from rewriting token.json, so a file that could not be rewritten kept the credential in cleartext while every command reported success. That file is now removed instead: signing in again costs one command, a stranded live credential costs the credential `lite logout` also reported a clean logout whenever the keyring package was missing, on the reasoning that an install without it could never have stored anything. The entry belongs to the OS, so a keychain-backed login survives a logout run from a venv without the cli extra. erase() now reports which keychain state applies, and logout warns with the advice that fixes each one, staying quiet for file-backed logins whose token file still carries its own secret Also pins the migration path's tightening of a world-readable legacy token.json, and moves the logout tests off patch() onto the injected vault --- litellm/litellm_core_utils/cli_keyring.py | 41 ++++++---- litellm/litellm_core_utils/cli_token_utils.py | 49 ++++++++++-- litellm/proxy/client/cli/commands/auth.py | 31 +++++--- tests/test_litellm/conftest.py | 13 +++- .../test_cli_token_utils.py | 77 ++++++++++++++++--- .../proxy/client/cli/test_auth_commands.py | 66 ++++++++++++++-- 6 files changed, 226 insertions(+), 51 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index b19b3d3bc83..0497991c2a0 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -36,6 +36,16 @@ class SecretStored: pass +@dataclass(frozen=True, slots=True) +class SecretErased: + pass + + +@dataclass(frozen=True, slots=True) +class SecretStranded: + pass + + @dataclass(frozen=True, slots=True) class KeyringNotInstalled: pass @@ -54,6 +64,7 @@ class KeyringUnreachable: KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable SecretRead: TypeAlias = SecretFound | SecretMissing | KeyringUnusable SecretWrite: TypeAlias = SecretStored | KeyringUnusable +SecretErase: TypeAlias = SecretErased | SecretStranded | KeyringUnusable class SecretVault(Protocol): @@ -63,7 +74,7 @@ class SecretVault(Protocol): def write(self, blob: str) -> SecretWrite: ... - def erase(self) -> bool: ... + def erase(self) -> SecretErase: ... class KeyringApi(Protocol): @@ -117,33 +128,31 @@ class KeyringVault: return KeyringUnreachable() return SecretStored() - def erase(self) -> bool: - """Whether the keychain is guaranteed to hold no credential afterwards. + def erase(self) -> SecretErase: + """Remove our entry, reporting whether the keychain is guaranteed to be free of it. - An uninstalled `keyring` package can never have stored one. A kill switch set after - a credential was stored leaves that entry out of reach, so erasure cannot be promised. + A keychain out of reach is never an erasure: the entry belongs to the OS, not to this + install, so it outlives an uninstalled `keyring` package and a kill switch set after login. + Those cases are reported apart from a confirmed entry that would not delete, because only + the caller knows whether this machine ever put a secret in a keychain. """ - if _import_keyring() is None: - return True - if _keyring_disabled(): - return False match self.read(): - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): - return False + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() as unusable: + return unusable case SecretMissing(): - return True + return SecretErased() case SecretFound(): return self._delete() - def _delete(self) -> bool: + def _delete(self) -> SecretErase: api: Final = _keyring_api() if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): - return False + return api try: api.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT) except Exception: # noqa: BLE001 # report the failure as a value so `lite logout` can warn - return False - return True + return SecretStranded() + return SecretErased() SYSTEM_KEYRING: Final[SecretVault] = KeyringVault() diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 40822e5b335..cd69f9470a3 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -25,9 +25,12 @@ from litellm.litellm_core_utils.cli_keyring import ( KeyringDisabled, KeyringNotInstalled, KeyringUnreachable, + SecretErase, + SecretErased, SecretFound, SecretMissing, SecretStored, + SecretStranded, SecretVault, SecretWrite, ) @@ -84,7 +87,7 @@ def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | N def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretWrite: - """Store a freshly minted credential. Reports whether the keychain took the secret, and why not""" + """Store a freshly minted credential. Reports where its secret material ended up, and why""" outcome: Final = ( SecretStored() if record.key is None @@ -94,11 +97,33 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN return outcome -def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> bool: - """Remove the credential from both stores. Returns whether the keychain is now free of it""" - erased: Final = vault.erase() +def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretErase: + """Remove the credential from both stores. Reports whether the keychain is now free of it""" + outcome: Final = vault.erase() + settled: Final = _nothing_left_behind(outcome) Path(get_cli_token_file_path()).unlink(missing_ok=True) - return erased + return SecretErased() if settled else outcome + + +def _nothing_left_behind(outcome: SecretErase) -> bool: + """Whether the keychain can be trusted to hold no credential of ours once the file is gone""" + match outcome: + case SecretErased(): + return True + case SecretStranded(): + return False + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): + return not _secret_lives_in_keychain() + + +def _secret_lives_in_keychain() -> bool: + """Whether the token file is the metadata half of a pair whose secret half went to a keychain. + + A file that still carries its own secret rules one out, which keeps `lite logout` quiet on the + machines that never had a keychain to begin with. + """ + record: Final = _read_token_file() + return record is not None and record.key is None and not record.jwt_token def get_litellm_gateway_api_key( @@ -200,10 +225,22 @@ def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliToken def _scrub_file_secret(record: CliTokenRecord) -> None: + """Leave no secret material in the token file once the vault holds it. + + A file that cannot be rewritten without the secret is removed instead. Signing in again costs + the user one command; a live credential left behind in cleartext costs them the credential. + """ if record.key is None and not record.jwt_token: return - with contextlib.suppress(OSError): + try: _write_token_file(_without_secret(record)) + except OSError: + _discard_token_file() + + +def _discard_token_file() -> None: + with contextlib.suppress(OSError): + Path(get_cli_token_file_path()).unlink(missing_ok=True) def _without_secret(record: CliTokenRecord) -> CliTokenRecord: diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 03906f9b6df..4cf18435473 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -17,7 +17,9 @@ from litellm.litellm_core_utils.cli_keyring import ( KeyringDisabled, KeyringNotInstalled, KeyringUnreachable, + SecretErased, SecretStored, + SecretStranded, SecretVault, SecretWrite, ) @@ -80,12 +82,16 @@ class CliAuthResult(TypedDict): team_id: str | None -KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( - "Your credential is stored in your OS keychain, which could not be read. Unlock it and retry, " - "install the keyring package with: pip install 'litellm[cli]', or run 'lite login'." +KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'" + +STRANDED_CREDENTIAL_MESSAGE: Final = ( + "Logged out locally, but your credential is still in the OS keychain and could not be removed." ) -KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'" +KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( + "Your credential is stored in your OS keychain, which could not be read. Unlock it, or install " + f"the keyring package with: {KEYRING_INSTALL_HINT}. Run 'lite login' to start over." +) def storage_notice(outcome: SecretWrite) -> str: @@ -742,11 +748,18 @@ def login(ctx: click.Context, config_claude: bool): @click.pass_context def logout(ctx: click.Context): """Logout and clear stored authentication""" - if clear_cli_token(vault=context_secret_vault(ctx)): - click.echo("Logged out successfully. Authentication token cleared.") - return - click.echo("Logged out. The local token file is gone, but the OS keychain entry could not be removed.") - click.echo("Unlock your keychain and run 'lite logout' again to clear it.") + match clear_cli_token(vault=context_secret_vault(ctx)): + case SecretErased(): + click.echo("Logged out successfully. Authentication token cleared.") + case KeyringNotInstalled(): + click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo(f"Install the keyring package with: {KEYRING_INSTALL_HINT}, then run 'lite logout' again.") + case KeyringDisabled(): + click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo(f"Unset {DISABLE_KEYRING_ENV_VAR} and run 'lite logout' again to clear it.") + case SecretStranded() | KeyringUnreachable(): + click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo("Unlock your keychain and run 'lite logout' again to clear it.") @click.command(name="print-token") diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index a716ec0e0aa..b42355fa045 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -25,10 +25,13 @@ from litellm._logging import ALL_LOGGERS from litellm.litellm_core_utils.cli_keyring import ( KeyringUnreachable, KeyringUnusable, + SecretErase, + SecretErased, SecretFound, SecretMissing, SecretRead, SecretStored, + SecretStranded, SecretWrite, ) from litellm.litellm_core_utils.prompt_templates import ( @@ -163,12 +166,14 @@ class FakeSecretVault: self.blob = blob return SecretStored() - def erase(self) -> bool: + def erase(self) -> SecretErase: self.erases += 1 - if not (self.available and self.erasable): - return False + if not self.available: + return self.failure + if not self.erasable: + return SecretStranded() if self.blob is not None else SecretErased() self.blob = None - return True + return SecretErased() @pytest.fixture diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 961d986e8f5..925b440dfb7 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -16,7 +16,9 @@ from litellm.litellm_core_utils.cli_keyring import ( KeyringDisabled, KeyringNotInstalled, KeyringUnreachable, + SecretErased, SecretStored, + SecretStranded, ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, @@ -126,6 +128,16 @@ class TestLoadCliToken: assert on_disk["user_email"] == "user@example.com" assert stat.S_IMODE(path.stat().st_mode) == 0o600 + def test_migration_tightens_a_world_readable_legacy_file(self, isolated_home, secret_vault_factory): + """An older `lite`, a loose umask, or a restored backup can leave token.json readable by + every account on the box. Migrating it must not preserve those permissions.""" + path = _write_legacy_file(isolated_home) + path.chmod(0o644) + + load_cli_token(vault=secret_vault_factory()) + + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + def test_legacy_file_survives_a_vault_that_refuses_to_store(self, isolated_home, secret_vault_factory): """Scrubbing the only copy of the secret after a failed keychain write would log the user out for good.""" @@ -304,12 +316,35 @@ class TestSaveCliToken: assert list(path.parent.glob(".tmp-*")) == [] +class TestScrubFailure: + """A keychain that took the secret while the file kept it is the worst of both stores: the + credential is live, it is in cleartext on disk, and every command reports success.""" + + def test_a_file_that_cannot_be_rewritten_is_removed_instead( + self, isolated_home, secret_vault_factory, monkeypatch + ): + path = _write_legacy_file(isolated_home) + vault = secret_vault_factory() + + def _explode(*args, **kwargs): + raise OSError("no space left on device") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert json.loads(vault.blob)["key"] == "sk-legacy" + assert not path.exists() + assert list(path.parent.glob(".tmp-*")) == [] + + class TestClearCliToken: def test_removes_the_credential_from_both_stores(self, isolated_home, secret_vault_factory): vault = secret_vault_factory() save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) - assert clear_cli_token(vault=vault) is True + assert clear_cli_token(vault=vault) == SecretErased() assert vault.blob is None assert not _token_file(isolated_home).exists() assert load_cli_token(vault=vault) is None @@ -318,11 +353,32 @@ class TestClearCliToken: _write_legacy_file(isolated_home) vault = secret_vault_factory(blob=_blob(), erasable=False) - assert clear_cli_token(vault=vault) is False + assert clear_cli_token(vault=vault) == SecretStranded() + assert not _token_file(isolated_home).exists() + + def test_logout_from_an_install_without_keyring_does_not_claim_the_keychain_is_clear( + self, isolated_home, secret_vault_factory + ): + """Log in where `litellm[cli]` is installed and the secret goes to the OS keychain; log out + from a venv without it and the entry survives, because it belongs to the OS rather than to + the package. Reporting a clean logout there leaves a live credential the user thinks is gone.""" + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) + + assert clear_cli_token(vault=vault) == KeyringNotInstalled() + assert not _token_file(isolated_home).exists() + + def test_a_file_backed_login_logs_out_quietly_without_keyring(self, isolated_home, secret_vault_factory): + """The complement: a user who never had a keychain keeps their whole credential in the file, + so removing it is a complete logout and must not warn about an entry that cannot exist.""" + _write_legacy_file(isolated_home) + vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) + + assert clear_cli_token(vault=vault) == SecretErased() assert not _token_file(isolated_home).exists() def test_is_safe_when_nothing_was_ever_stored(self, isolated_home, secret_vault_factory): - assert clear_cli_token(vault=secret_vault_factory()) is True + assert clear_cli_token(vault=secret_vault_factory()) == SecretErased() class TestIsCliTokenFresh: @@ -384,7 +440,7 @@ class TestKeyringVault: assert vault.write("blob-1") == SecretStored() assert vault.read() == SecretFound("blob-1") - assert vault.erase() is True + assert vault.erase() == SecretErased() assert vault.read() == SecretMissing() assert {call[1:] for call in fake.calls} == {(KEYRING_SERVICE, KEYRING_ACCOUNT)} @@ -392,24 +448,25 @@ class TestKeyringVault: """`LITELLM_CLI_DISABLE_KEYRING` has to work without importing keyring, because keyring caches its backend on first use and cannot be reconfigured later. Erase still fails: a credential stored before the switch was set may be in the keychain, and with reads - disabled `lite logout` cannot verify it is gone, so it must warn instead.""" + disabled `lite logout` cannot verify it is gone, so it must say so instead.""" monkeypatch.setenv(DISABLE_KEYRING_ENV_VAR, "1") vault = KeyringVault() assert vault.read() == KeyringDisabled() assert vault.write("blob-1") == KeyringDisabled() - assert vault.erase() is False + assert vault.erase() == KeyringDisabled() def test_an_uninstalled_keyring_library_degrades_to_the_file(self, monkeypatch): """keyring is an optional extra, so the SDK must survive its absence rather than raise on - the hot path.""" + the hot path. Erase cannot succeed: the entry belongs to the OS and outlives the package, + so an install without it is not evidence that the keychain is empty.""" monkeypatch.delenv(DISABLE_KEYRING_ENV_VAR, raising=False) monkeypatch.setitem(sys.modules, "keyring", None) vault = KeyringVault() assert vault.read() == KeyringNotInstalled() assert vault.write("blob-1") == KeyringNotInstalled() - assert vault.erase() is True + assert vault.erase() == KeyringNotInstalled() def test_a_locked_keychain_is_reported_not_raised(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("keyring is locked"))) @@ -424,9 +481,9 @@ class TestKeyringVault: def test_a_refused_delete_is_reported_so_logout_can_warn(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(stored="blob-1", delete_error=RuntimeError("locked"))) - assert KeyringVault().erase() is False + assert KeyringVault().erase() == SecretStranded() def test_erasing_a_locked_keychain_is_a_failure(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("locked"))) - assert KeyringVault().erase() is False + assert KeyringVault().erase() == KeyringUnreachable() diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 1f2d48f6547..33bd8307c21 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -46,6 +46,12 @@ def _write_home_json(home: Path, filename: str, payload: dict[str, object]) -> N (litellm_dir / filename).write_text(json.dumps(payload)) +def _write_token_file(home: Path, *, key: str | None) -> None: + """A stored login: `key=None` is the metadata half of a keychain-backed pair, a key is a file-backed one.""" + payload: dict[str, object] = {"base_url": "https://test.example.com", "user_id": "u-1", "timestamp": time.time()} + _write_home_json(home, "token.json", payload if key is None else {**payload, "key": key}) + + def _secret_blob(base_url: str, key: str) -> str: return json.dumps({"base_url": base_url, "key": key, "jwt_token": ""}) @@ -427,14 +433,62 @@ class TestLogoutCommand: """Setup for each test""" self.runner = CliRunner() - def test_logout_success(self): + def test_logout_success(self, isolated_home, secret_vault_factory): """Test successful logout""" - with patch("litellm.proxy.client.cli.commands.auth.clear_cli_token") as mock_clear: - result = self.runner.invoke(logout) + vault = secret_vault_factory(blob=_secret_blob("https://test.example.com", "sk-stored")) + _write_token_file(isolated_home, key=None) - assert result.exit_code == 0 - assert "Logged out successfully" in result.output - mock_clear.assert_called_once() + result = self.runner.invoke(logout, obj={"secret_vault": vault}) + + assert result.exit_code == 0 + assert "Logged out successfully" in result.output + assert vault.blob is None + assert not (isolated_home / ".litellm" / "token.json").exists() + + def test_logout_without_the_keyring_package_does_not_claim_the_keychain_is_clear( + self, isolated_home, secret_vault_factory + ): + """Logging out from an install without the cli extra cannot touch an entry a keychain-backed + login left behind, so it must point at the package rather than report a clean logout.""" + _write_token_file(isolated_home, key=None) + + result = self.runner.invoke( + logout, obj={"secret_vault": secret_vault_factory(available=False, failure=KeyringNotInstalled())} + ) + + assert result.exit_code == 0 + assert "Logged out successfully" not in result.output + assert "still in the OS keychain" in result.output + assert "pip install 'litellm[cli]'" in result.output + + def test_logout_warns_when_the_keychain_refuses_to_release_the_entry( + self, isolated_home, secret_vault_factory + ): + """A locked keychain leaves a live credential behind that the user believes is gone.""" + vault = secret_vault_factory( + blob=_secret_blob("https://test.example.com", "sk-stored"), erasable=False + ) + _write_token_file(isolated_home, key=None) + + result = self.runner.invoke(logout, obj={"secret_vault": vault}) + + assert result.exit_code == 0 + assert "Logged out successfully" not in result.output + assert "still in the OS keychain" in result.output + assert "Unlock your keychain" in result.output + + def test_logout_from_a_file_only_login_stays_quiet(self, isolated_home, secret_vault_factory): + """The credential never went to a keychain, so removing the file is the whole logout and + warning about a keychain entry would send the user chasing one that cannot exist.""" + _write_token_file(isolated_home, key="sk-in-file") + + result = self.runner.invoke( + logout, obj={"secret_vault": secret_vault_factory(available=False, failure=KeyringNotInstalled())} + ) + + assert result.exit_code == 0 + assert "Logged out successfully" in result.output + assert "still in the OS keychain" not in result.output class TestWhoamiCommand: From 424e74ba9ffc9f33757d3c68f90d0fef2cda4079 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:31:43 -0700 Subject: [PATCH 07/53] fix(cli): roll the keychain write back when the plaintext copy cannot be removed Removing the file when it could not be rewritten covered a full disk, but not a ~/.litellm that permits neither the rewrite nor the delete, which is what a `sudo lite login` leaves behind. There the secret was copied into the keychain and kept in cleartext on disk, so migration widened exposure instead of narrowing it Migration now only keeps the vault copy if the file's copy is gone. When it is not, the write is rolled back and the user is left exactly as they were, logged in with one copy of the credential --- litellm/litellm_core_utils/cli_token_utils.py | 26 +++++++++++++------ .../test_cli_token_utils.py | 20 ++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index cd69f9470a3..53289770c1a 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -12,7 +12,6 @@ first time it reads one. This module has no dependencies on proxy code and can be safely imported at the SDK level. """ -import contextlib import time from pathlib import Path from types import MappingProxyType @@ -217,30 +216,41 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) - def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: + """Move a file-held secret into the vault, but only if the file's copy can be taken away. + + Migrating without scrubbing would leave the credential live in two stores instead of one, so a + file that will not give its copy up rolls the vault write back rather than widening exposure. + """ if record.key is None: return None - if isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): - _scrub_file_secret(record) + if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): + return record + if not _scrub_file_secret(record): + vault.erase() return record -def _scrub_file_secret(record: CliTokenRecord) -> None: +def _scrub_file_secret(record: CliTokenRecord) -> bool: """Leave no secret material in the token file once the vault holds it. A file that cannot be rewritten without the secret is removed instead. Signing in again costs the user one command; a live credential left behind in cleartext costs them the credential. """ if record.key is None and not record.jwt_token: - return + return True try: _write_token_file(_without_secret(record)) except OSError: - _discard_token_file() + return _discard_token_file() + return True -def _discard_token_file() -> None: - with contextlib.suppress(OSError): +def _discard_token_file() -> bool: + try: Path(get_cli_token_file_path()).unlink(missing_ok=True) + except OSError: + return False + return True def _without_secret(record: CliTokenRecord) -> CliTokenRecord: diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 925b440dfb7..cbc93bbde71 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -1,4 +1,5 @@ import json +import os import stat import sys import time @@ -320,6 +321,25 @@ class TestScrubFailure: """A keychain that took the secret while the file kept it is the worst of both stores: the credential is live, it is in cleartext on disk, and every command reports success.""" + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") + def test_a_file_that_will_not_give_its_copy_up_rolls_the_vault_write_back( + self, isolated_home, secret_vault_factory + ): + """Handing the keychain a copy without taking the file's away leaves the credential live in + two stores instead of one. A directory that permits neither the rewrite nor the delete, a + root-owned ~/.litellm left behind by a `sudo lite login`, must widen nothing.""" + path = _write_legacy_file(isolated_home) + vault = secret_vault_factory() + path.parent.chmod(0o500) + try: + record = load_cli_token(vault=vault) + finally: + path.parent.chmod(0o700) + + assert record.key == "sk-legacy" + assert json.loads(path.read_text())["key"] == "sk-legacy" + assert vault.blob is None + def test_a_file_that_cannot_be_rewritten_is_removed_instead( self, isolated_home, secret_vault_factory, monkeypatch ): From 3c73a39877fa97db3dff4e22dc685bcb77fa4041 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:10:59 -0700 Subject: [PATCH 08/53] fix(cli): verify every credential store transition before reporting it done A keyring backend can accept a write and keep nothing. That is exactly what `keyring --disable` and PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring select, and it raises nothing to distinguish itself, so `lite login` was handing the credential to a black hole, scrubbing its own copy from token.json, and printing a success message over a login that no longer worked. Reading the value back is the only way to tell that backend apart from a keychain that really stored the secret. The same rule closes the rest of the gaps. A credential the token file will not record is taken back out of the keychain instead of being left live on a machine with no record of it, and is reported rather than raised. The migration stages its scrubbed file before the keychain is handed anything, so a directory that will not accept the rewrite stops the move rather than leaving the secret in two places. Logout no longer reads a key in the file as proof that the keychain is clear, which was never sound across two separate runs, and only draws that conclusion when the `keyring` package is missing outright, where nothing could have reached a keychain at all. --- litellm/litellm_core_utils/cli_keyring.py | 26 +++- litellm/litellm_core_utils/cli_token_utils.py | 104 +++++++++----- litellm/litellm_core_utils/private_json.py | 32 ++++- litellm/proxy/client/cli/commands/auth.py | 63 +++++++-- .../test_cli_token_utils.py | 131 ++++++++++++++++-- .../proxy/client/cli/test_auth_commands.py | 62 ++++++++- 6 files changed, 356 insertions(+), 62 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index 0497991c2a0..15282fc522c 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -6,8 +6,12 @@ Linux Secret Service) that holds the credential minted by `lite login`. The `keyring` package is optional and imported lazily, so importing this module never pulls it in. Every failure is returned as a value, naming which of the -three ways the keychain can be out of reach applies, so callers can degrade to -the token file and tell the user what to do about it. +ways the keychain can be out of reach applies, so callers can degrade to the +token file and tell the user what to do about it. + +A write is only reported as stored once it has been read back, because keyring's +null backend, which `keyring --disable` and headless CI images both select, +accepts every write and keeps nothing. """ import os @@ -61,7 +65,12 @@ class KeyringUnreachable: pass -KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable +@dataclass(frozen=True, slots=True) +class KeyringDiscardsWrites: + pass + + +KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable | KeyringDiscardsWrites SecretRead: TypeAlias = SecretFound | SecretMissing | KeyringUnusable SecretWrite: TypeAlias = SecretStored | KeyringUnusable SecretErase: TypeAlias = SecretErased | SecretStranded | KeyringUnusable @@ -119,6 +128,13 @@ class KeyringVault: return SecretMissing() if blob is None else SecretFound(blob) def write(self, blob: str) -> SecretWrite: + """Store the secret, reporting stored only once the keychain hands the same bytes back. + + A backend that accepts writes and keeps nothing, which is exactly what `keyring --disable` + and `PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring` select, raises nothing to + distinguish itself. Reading the value back is the only way to tell it apart from a keychain + that really stored the credential, and the caller is about to drop its own copy on our word. + """ api: Final = _keyring_api() if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): return api @@ -126,7 +142,7 @@ class KeyringVault: api.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, blob) except Exception: # noqa: BLE001 # a keychain that refuses the write falls back to the token file return KeyringUnreachable() - return SecretStored() + return SecretStored() if self.read() == SecretFound(blob) else KeyringDiscardsWrites() def erase(self) -> SecretErase: """Remove our entry, reporting whether the keychain is guaranteed to be free of it. @@ -137,7 +153,7 @@ class KeyringVault: the caller knows whether this machine ever put a secret in a keychain. """ match self.read(): - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() as unusable: + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites() as unusable: return unusable case SecretMissing(): return SecretErased() diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 53289770c1a..af1b918fc2a 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -13,15 +13,17 @@ This module has no dependencies on proxy code and can be safely imported at the """ import time +from dataclasses import dataclass from pathlib import Path from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias from pydantic import BaseModel, ConfigDict, ValidationError from litellm.litellm_core_utils.cli_keyring import ( SYSTEM_KEYRING, KeyringDisabled, + KeyringDiscardsWrites, KeyringNotInstalled, KeyringUnreachable, SecretErase, @@ -33,7 +35,23 @@ from litellm.litellm_core_utils.cli_keyring import ( SecretVault, SecretWrite, ) -from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json +from litellm.litellm_core_utils.private_json import ( + commit_staged_json, + discard_staged_json, + ensure_private_dir, + stage_private_json, + write_private_json, +) + + +@dataclass(frozen=True, slots=True) +class CredentialNotSaved: + """The credential was minted but no store would keep it, so this machine has none.""" + + detail: str + + +SecretSave: TypeAlias = SecretWrite | CredentialNotSaved class CliTokenRecord(BaseModel): @@ -85,14 +103,24 @@ def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | N return _resolve_secret(record, vault) -def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretWrite: - """Store a freshly minted credential. Reports where its secret material ended up, and why""" +def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretSave: + """Store a freshly minted credential. Reports where its secret material ended up, and why. + + The token file is what makes a keychain-backed credential findable again, so a file that will + not be written takes the keychain copy down with it rather than leaving a live credential + stored under a machine that has no record of it. + """ outcome: Final = ( SecretStored() if record.key is None else vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)) ) - _write_token_file(_without_secret(record) if isinstance(outcome, SecretStored) else record) + try: + _write_token_file(_without_secret(record) if isinstance(outcome, SecretStored) else record) + except OSError as error: + if record.key is not None and isinstance(outcome, SecretStored): + vault.erase() + return CredentialNotSaved(str(error)) return outcome @@ -105,24 +133,30 @@ def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretErase: def _nothing_left_behind(outcome: SecretErase) -> bool: - """Whether the keychain can be trusted to hold no credential of ours once the file is gone""" + """Whether the keychain can be trusted to hold no credential of ours once the file is gone. + + A keychain that exists but is out of reach right now is never trusted, whatever the token file + looks like: the login that stored a secret there and the logout that cannot remove it are + separate runs, free to differ in whether the keychain was usable at the time. + """ match outcome: case SecretErased(): return True - case SecretStranded(): + case SecretStranded() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites(): return False - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): - return not _secret_lives_in_keychain() + case KeyringNotInstalled(): + return _file_holds_its_own_secret() -def _secret_lives_in_keychain() -> bool: - """Whether the token file is the metadata half of a pair whose secret half went to a keychain. +def _file_holds_its_own_secret() -> bool: + """Whether the stored login keeps its secret in the token file, ruling out a keychain entry. - A file that still carries its own secret rules one out, which keeps `lite logout` quiet on the - machines that never had a keychain to begin with. + Sound only against a missing `keyring` package, the one way to lose the keychain that had to + hold at storage time too, since nothing here can reach a keychain without it. A file whose + secret half is absent went to a keychain by definition, and so rules nothing out. """ record: Final = _read_token_file() - return record is not None and record.key is None and not record.jwt_token + return record is not None and record.key is not None def get_litellm_gateway_api_key( @@ -179,7 +213,7 @@ def is_cli_token_fresh(token_data: CliTokenRecord, buffer_hours: float = 0.1) -> def _read_token_file() -> CliTokenRecord | None: try: raw: Final = Path(get_cli_token_file_path()).read_text() - except OSError: + except (OSError, ValueError): return None try: return CliTokenRecord.model_validate_json(raw) @@ -193,7 +227,7 @@ def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecor return _apply_vault_secret(record, blob, vault) case SecretMissing(): return _migrate_file_secret(record, vault) - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites(): return record @@ -216,38 +250,46 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) - def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: - """Move a file-held secret into the vault, but only if the file's copy can be taken away. + """Move a file-held secret into the vault, but only once the file's copy can be taken away. - Migrating without scrubbing would leave the credential live in two stores instead of one, so a - file that will not give its copy up rolls the vault write back rather than widening exposure. + The scrubbed file is staged first so a directory that will not accept it stops the migration + before the keychain is handed anything. Copying the credential into a second store and only + then discovering the first one cannot be cleaned would widen exposure instead of narrowing it, + which is the opposite of what moving it into the keychain is for. """ if record.key is None: return None - if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): + staged: Final = _stage_scrubbed_file(record) + if staged is None: return record - if not _scrub_file_secret(record): + if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): + discard_staged_json(staged) + return record + if not _commit_scrubbed_file(staged): vault.erase() return record def _scrub_file_secret(record: CliTokenRecord) -> bool: - """Leave no secret material in the token file once the vault holds it. - - A file that cannot be rewritten without the secret is removed instead. Signing in again costs - the user one command; a live credential left behind in cleartext costs them the credential. - """ + """Leave no secret material in the token file once the vault holds it""" if record.key is None and not record.jwt_token: return True + staged: Final = _stage_scrubbed_file(record) + return staged is not None and _commit_scrubbed_file(staged) + + +def _stage_scrubbed_file(record: CliTokenRecord) -> str | None: + path: Final = Path(get_cli_token_file_path()) try: - _write_token_file(_without_secret(record)) + ensure_private_dir(path.parent) + return stage_private_json(str(path), _without_secret(record).model_dump(exclude_none=True)) except OSError: - return _discard_token_file() - return True + return None -def _discard_token_file() -> bool: +def _commit_scrubbed_file(staged: str) -> bool: try: - Path(get_cli_token_file_path()).unlink(missing_ok=True) + commit_staged_json(staged, get_cli_token_file_path()) except OSError: return False return True diff --git a/litellm/litellm_core_utils/private_json.py b/litellm/litellm_core_utils/private_json.py index 32bc2e169e2..fbeb74aab5a 100644 --- a/litellm/litellm_core_utils/private_json.py +++ b/litellm/litellm_core_utils/private_json.py @@ -16,8 +16,12 @@ def ensure_private_dir(directory: Path) -> None: directory.chmod(PRIVATE_DIR_MODE) -def write_private_json(path: str, data: Mapping[str, object]) -> None: - """Atomically write JSON to path with owner-only permissions (0600)""" +def stage_private_json(path: str, data: Mapping[str, object]) -> str: + """Write JSON to a private temp file beside `path`, ready for `commit_staged_json`. + + Staging is the half that can fail on a read-only or full directory, so callers with something + to lose can find that out before they act on the assumption that the rewrite will land. + """ parent: Final = Path(path).parent parent.mkdir(parents=True, exist_ok=True) fd, tmp_path = tempfile.mkstemp(dir=str(parent), prefix=".tmp-", suffix=".json") @@ -26,6 +30,26 @@ def write_private_json(path: str, data: Mapping[str, object]) -> None: json.dump(data, f, indent=2) f.flush() os.fsync(f.fileno()) - os.replace(tmp_path, path) - finally: + except BaseException: Path(tmp_path).unlink(missing_ok=True) + raise + return tmp_path + + +def commit_staged_json(staged: str, path: str) -> None: + """Move a staged file into place, replacing whatever is there in one step""" + try: + os.replace(staged, path) + except OSError: + Path(staged).unlink(missing_ok=True) + raise + + +def discard_staged_json(staged: str) -> None: + """Throw a staged file away when the change it was part of is abandoned""" + Path(staged).unlink(missing_ok=True) + + +def write_private_json(path: str, data: Mapping[str, object]) -> None: + """Atomically write JSON to path with owner-only permissions (0600)""" + commit_staged_json(stage_private_json(path, data), path) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 4cf18435473..eba9994f7ec 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -15,16 +15,20 @@ from litellm.litellm_core_utils.cli_keyring import ( DISABLE_KEYRING_ENV_VAR, SYSTEM_KEYRING, KeyringDisabled, + KeyringDiscardsWrites, KeyringNotInstalled, KeyringUnreachable, SecretErased, + SecretFound, + SecretMissing, SecretStored, SecretStranded, SecretVault, - SecretWrite, ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, + CredentialNotSaved, + SecretSave, clear_cli_token, get_cli_token_file_path, get_litellm_gateway_api_key, @@ -84,17 +88,19 @@ class CliAuthResult(TypedDict): KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'" +KEYRING_ENABLE_HINT: Final = "keyring --enable (or unset PYTHON_KEYRING_BACKEND)" + STRANDED_CREDENTIAL_MESSAGE: Final = ( "Logged out locally, but your credential is still in the OS keychain and could not be removed." ) -KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( - "Your credential is stored in your OS keychain, which could not be read. Unlock it, or install " - f"the keyring package with: {KEYRING_INSTALL_HINT}. Run 'lite login' to start over." +UNCHECKED_KEYCHAIN_MESSAGE: Final = ( + "Logged out locally, but your OS keychain could not be checked, so a credential stored there by " + "an earlier login may still be usable." ) -def storage_notice(outcome: SecretWrite) -> str: +def storage_notice(outcome: SecretSave) -> str: """Tell the user where the credential ended up, and how to get keychain storage if it did not.""" path: Final = get_cli_token_file_path() match outcome: @@ -109,6 +115,38 @@ def storage_notice(outcome: SecretWrite) -> str: return f"Keychain storage is off ({DISABLE_KEYRING_ENV_VAR}). Credential stored in {path} (owner-only)." case KeyringUnreachable(): return f"No OS keychain available. Credential stored in {path} (owner-only)." + case KeyringDiscardsWrites(): + return ( + f"Your keyring backend keeps nothing it is given, so the credential was stored in {path} " + f"(owner-only) instead. For OS keychain storage, run: {KEYRING_ENABLE_HINT}" + ) + case CredentialNotSaved(detail=detail): + return ( + f"Signed in, but the credential could not be saved to {path}: {detail}. " + "Nothing was kept, so run 'lite login' again once that path is writable." + ) + + +def keychain_unreadable_notice(vault: SecretVault) -> str: + """Explain why the secret half of a stored login cannot be produced, and what fixes it""" + match vault.read(): + case KeyringNotInstalled(): + return ( + "Your credential is in your OS keychain, which this install cannot read without the " + f"keyring package. Install it with: {KEYRING_INSTALL_HINT}, or run 'lite login' to start over." + ) + case KeyringDisabled(): + return ( + f"Your credential is in your OS keychain, which {DISABLE_KEYRING_ENV_VAR} is blocking. " + "Unset it, or run 'lite login' to start over." + ) + case KeyringUnreachable() | KeyringDiscardsWrites(): + return ( + "Your credential is in your OS keychain, which could not be read. Unlock it, or run " + "'lite login' to start over." + ) + case SecretFound() | SecretMissing(): + return "Your credential could not be read from your OS keychain. Run 'lite login' to start over." def context_secret_vault(ctx: click.Context) -> SecretVault: @@ -715,6 +753,8 @@ def login(ctx: click.Context, config_claude: bool): click.echo("\nLogin successful!") click.echo(f"JWT Token: {api_key[:20]}...") click.echo(storage_notice(stored)) + if isinstance(stored, CredentialNotSaved): + return click.echo("You can now use the CLI without specifying --api-key") if config_claude: @@ -751,14 +791,17 @@ def logout(ctx: click.Context): match clear_cli_token(vault=context_secret_vault(ctx)): case SecretErased(): click.echo("Logged out successfully. Authentication token cleared.") + case SecretStranded(): + click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo("Unlock your keychain and run 'lite logout' again to clear it.") case KeyringNotInstalled(): click.echo(STRANDED_CREDENTIAL_MESSAGE) click.echo(f"Install the keyring package with: {KEYRING_INSTALL_HINT}, then run 'lite logout' again.") case KeyringDisabled(): - click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo(UNCHECKED_KEYCHAIN_MESSAGE) click.echo(f"Unset {DISABLE_KEYRING_ENV_VAR} and run 'lite logout' again to clear it.") - case SecretStranded() | KeyringUnreachable(): - click.echo(STRANDED_CREDENTIAL_MESSAGE) + case KeyringUnreachable() | KeyringDiscardsWrites(): + click.echo(UNCHECKED_KEYCHAIN_MESSAGE) click.echo("Unlock your keychain and run 'lite logout' again to clear it.") @@ -795,7 +838,7 @@ def print_token(ctx: click.Context): api_key: Final = token_data.key if not api_key: - click.echo(KEYCHAIN_UNREACHABLE_MESSAGE, err=True) + click.echo(keychain_unreadable_notice(context_secret_vault(ctx)), err=True) sys.exit(1) click.echo(api_key) @@ -821,7 +864,7 @@ def whoami(ctx: click.Context): click.echo(f"Token age: {age_hours:.1f} hours") if token_data.key is None: - click.echo(KEYCHAIN_UNREACHABLE_MESSAGE) + click.echo(keychain_unreadable_notice(context_secret_vault(ctx))) if age_hours > CLI_JWT_EXPIRATION_HOURS: click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.") diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index cbc93bbde71..e0f5f99dc1b 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.cli_keyring import ( SecretFound, SecretMissing, KeyringDisabled, + KeyringDiscardsWrites, KeyringNotInstalled, KeyringUnreachable, SecretErased, @@ -23,6 +24,7 @@ from litellm.litellm_core_utils.cli_keyring import ( ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, + CredentialNotSaved, clear_cli_token, get_cli_token_file_path, get_litellm_gateway_api_key, @@ -225,6 +227,15 @@ class TestLoadCliToken: assert record.key == "sk-legacy" + def test_a_token_file_that_is_not_text_is_not_a_login(self, isolated_home, secret_vault_factory): + """A truncated write or a half-synced backup can leave bytes that are not UTF-8 at all. + Reading them must fail the way an absent file does, not crash every `lite` command.""" + path = _token_file(isolated_home) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\xff\xfe not utf-8 at all") + + assert load_cli_token(vault=secret_vault_factory()) is None + def test_corrupt_token_file_is_not_a_login(self, isolated_home, secret_vault_factory): _token_file(isolated_home).parent.mkdir() _token_file(isolated_home).write_text("not json at all {{{") @@ -301,6 +312,40 @@ class TestSaveCliToken: assert stat.S_IMODE(config_dir.stat().st_mode) == 0o700 + def test_a_credential_no_store_would_keep_is_reported_rather_than_raised( + self, isolated_home, secret_vault_factory, monkeypatch + ): + """`lite login` catches whatever escapes here and calls it an authentication failure, which + is the one thing that did not happen: the proxy minted a real credential. Saying so lets the + user act on the actual problem instead of retrying a sign-in that already worked.""" + + def _explode(*args, **kwargs): + raise OSError("read-only file system") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + + outcome = save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=secret_vault_factory()) + + assert isinstance(outcome, CredentialNotSaved) + assert "read-only file system" in outcome.detail + + def test_a_credential_the_file_will_not_record_is_taken_back_out_of_the_keychain( + self, isolated_home, secret_vault_factory, monkeypatch + ): + """The token file is what makes a keychain entry findable again. Leaving the secret in the + keychain with nothing pointing at it strands a live credential under a machine that has no + idea it is there, and no `lite logout` would ever go looking for it.""" + vault = secret_vault_factory() + + def _explode(*args, **kwargs): + raise OSError("read-only file system") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) + + assert vault.blob is None + def test_a_failed_write_leaves_the_previous_credential_intact(self, isolated_home, secret_vault_factory, monkeypatch): path = _write_legacy_file(isolated_home) before = path.read_text() @@ -340,9 +385,11 @@ class TestScrubFailure: assert json.loads(path.read_text())["key"] == "sk-legacy" assert vault.blob is None - def test_a_file_that_cannot_be_rewritten_is_removed_instead( + def test_a_full_disk_stops_the_migration_before_the_keychain_is_handed_anything( self, isolated_home, secret_vault_factory, monkeypatch ): + """The scrubbed file is staged first precisely so this is knowable in advance. A disk that + cannot take the rewrite leaves the credential where it already was, in one store.""" path = _write_legacy_file(isolated_home) vault = secret_vault_factory() @@ -354,8 +401,8 @@ class TestScrubFailure: record = load_cli_token(vault=vault) assert record.key == "sk-legacy" - assert json.loads(vault.blob)["key"] == "sk-legacy" - assert not path.exists() + assert vault.blob is None + assert json.loads(path.read_text())["key"] == "sk-legacy" assert list(path.parent.glob(".tmp-*")) == [] @@ -376,12 +423,39 @@ class TestClearCliToken: assert clear_cli_token(vault=vault) == SecretStranded() assert not _token_file(isolated_home).exists() + @pytest.mark.parametrize( + "failure", [KeyringDisabled(), KeyringUnreachable(), KeyringDiscardsWrites()] + ) + def test_a_secret_in_the_file_is_no_evidence_about_a_keychain_that_exists( + self, isolated_home, secret_vault_factory, failure + ): + """Store a secret in the keychain, sign in again while the keychain is unusable so the new + secret lands in the file, then log out while it is still unusable. The file now carries its + own secret and the first login's entry is still there, so reading the file as proof of a + clean keychain reports a logout that did not happen.""" + _write_legacy_file(isolated_home) + vault = secret_vault_factory(available=False, failure=failure) + + assert clear_cli_token(vault=vault) == failure + assert not _token_file(isolated_home).exists() + + def test_a_second_logout_still_reports_the_keychain_it_could_not_clear( + self, isolated_home, secret_vault_factory + ): + """The first logout deletes the file and tells the user to run it again once the keychain is + reachable. If the second run reads that missing file as proof of a clean keychain, the advice + turns into the very false all-clear it was issued to prevent.""" + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(available=False, failure=KeyringUnreachable()) + + assert clear_cli_token(vault=vault) == KeyringUnreachable() + assert clear_cli_token(vault=vault) == KeyringUnreachable() + def test_logout_from_an_install_without_keyring_does_not_claim_the_keychain_is_clear( self, isolated_home, secret_vault_factory ): - """Log in where `litellm[cli]` is installed and the secret goes to the OS keychain; log out - from a venv without it and the entry survives, because it belongs to the OS rather than to - the package. Reporting a clean logout there leaves a live credential the user thinks is gone.""" + """A file holding only metadata put its secret in a keychain by definition. Losing the + package that reaches it does not take the entry with it, so this cannot report success.""" _write_metadata_only_file(isolated_home) vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) @@ -389,8 +463,9 @@ class TestClearCliToken: assert not _token_file(isolated_home).exists() def test_a_file_backed_login_logs_out_quietly_without_keyring(self, isolated_home, secret_vault_factory): - """The complement: a user who never had a keychain keeps their whole credential in the file, - so removing it is a complete logout and must not warn about an entry that cannot exist.""" + """The complement, and the one inference the file does support: nothing here can reach a + keychain without the package, so an install that lacks it and a file that still holds its + own secret between them account for the whole credential.""" _write_legacy_file(isolated_home) vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) @@ -417,11 +492,12 @@ class TestIsCliTokenFresh: class _FakeKeyringModule: - def __init__(self, stored=None, *, get_error=None, set_error=None, delete_error=None): + def __init__(self, stored=None, *, get_error=None, set_error=None, delete_error=None, discard=False): self.stored = stored self.get_error = get_error self.set_error = set_error self.delete_error = delete_error + self.discard = discard self.calls = [] def get_password(self, service_name, username): @@ -434,6 +510,8 @@ class _FakeKeyringModule: self.calls.append(("set", service_name, username)) if self.set_error is not None: raise self.set_error + if self.discard: + return self.stored = password def delete_password(self, service_name, username): @@ -503,6 +581,41 @@ class TestKeyringVault: assert KeyringVault().erase() == SecretStranded() + def test_a_backend_that_keeps_nothing_is_not_a_successful_write(self, install_fake_keyring): + """keyring's null backend accepts every write, stores nothing, and raises nothing to say so. + Taking its silence for success is how a credential gets deleted: the caller drops its own + copy on our word. Only reading the value back tells the two apart.""" + fake = install_fake_keyring(_FakeKeyringModule(discard=True)) + + assert KeyringVault().write("blob-1") == KeyringDiscardsWrites() + assert fake.stored is None + + def test_the_real_null_backend_is_rejected(self, monkeypatch): + """Pinned against the actual library rather than the double above, because the whole risk is + that upstream's no-op write looks exactly like a successful one.""" + keyring = pytest.importorskip("keyring") + null_backend = pytest.importorskip("keyring.backends.null") + monkeypatch.delenv(DISABLE_KEYRING_ENV_VAR, raising=False) + previous = keyring.get_keyring() + keyring.set_keyring(null_backend.Keyring()) + try: + assert KeyringVault().write("blob-1") == KeyringDiscardsWrites() + finally: + keyring.set_keyring(previous) + + def test_a_credential_survives_a_backend_that_keeps_nothing( + self, isolated_home, install_fake_keyring + ): + """The end of the same story: the credential must still be usable afterwards. Reporting the + discard is only worth anything if the token file then keeps the copy the keychain refused.""" + install_fake_keyring(_FakeKeyringModule(discard=True)) + + outcome = save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-only-copy")) + + assert outcome == KeyringDiscardsWrites() + assert json.loads(_token_file(isolated_home).read_text())["key"] == "sk-only-copy" + assert load_cli_token().key == "sk-only-copy" + def test_erasing_a_locked_keychain_is_a_failure(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("locked"))) diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 33bd8307c21..8e1551c0720 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -16,12 +16,13 @@ from litellm.constants import CLI_JWT_EXPIRATION_HOURS from litellm.litellm_core_utils.cli_keyring import ( DISABLE_KEYRING_ENV_VAR, KeyringDisabled, + KeyringDiscardsWrites, KeyringNotInstalled, ) from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord, save_cli_token from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.auth import ( - KEYCHAIN_UNREACHABLE_MESSAGE, + DISABLE_KEYRING_ENV_VAR, get_stored_api_key, login, logout, @@ -461,6 +462,20 @@ class TestLogoutCommand: assert "still in the OS keychain" in result.output assert "pip install 'litellm[cli]'" in result.output + def test_logout_does_not_call_an_unusable_keychain_clean(self, isolated_home, secret_vault_factory): + """A keychain-backed login, then a login that fell back to the file because the keychain had + become unusable, leaves the first entry live. The file's own secret says nothing about it, + so a clean bill of health here is the one answer that cannot be justified.""" + _write_token_file(isolated_home, key="sk-in-file") + vault = secret_vault_factory(available=False, failure=KeyringDisabled()) + + result = self.runner.invoke(logout, obj={"secret_vault": vault}) + + assert result.exit_code == 0 + assert "Logged out successfully" not in result.output + assert "could not be checked" in result.output + assert DISABLE_KEYRING_ENV_VAR in result.output + def test_logout_warns_when_the_keychain_refuses_to_release_the_entry( self, isolated_home, secret_vault_factory ): @@ -1003,6 +1018,19 @@ class TestKeychainBackedCommands: assert "No OS keychain available" not in result.output assert json.loads(token_file.read_text())["key"] == "sk-minted" + def test_login_keeps_the_credential_when_the_backend_keeps_nothing( + self, isolated_home, secret_vault_factory + ): + """A backend that accepts writes and stores nothing must not be reported as keychain + storage, because the file is then told to drop the only remaining copy.""" + result = self._login(secret_vault_factory(available=False, failure=KeyringDiscardsWrites())) + + token_file = isolated_home / ".litellm" / "token.json" + assert result.exit_code == 0 + assert "Credential stored in your OS keychain." not in result.output + assert "keyring --enable" in result.output + assert json.loads(token_file.read_text())["key"] == "sk-minted" + def test_login_names_the_kill_switch_instead_of_blaming_the_machine( self, isolated_home, secret_vault_factory ): @@ -1061,7 +1089,8 @@ class TestKeychainBackedCommands: result = self.runner.invoke(print_token, obj=obj) assert result.exit_code == 1 - assert KEYCHAIN_UNREACHABLE_MESSAGE in result.output + assert "could not be read" in result.output + assert "lite login" in result.output def test_whoami_flags_a_locked_keychain(self, isolated_home, secret_vault_factory): _write_home_json( @@ -1074,7 +1103,34 @@ class TestKeychainBackedCommands: result = self.runner.invoke(whoami, obj=obj) assert "Authenticated" in result.output - assert KEYCHAIN_UNREACHABLE_MESSAGE in result.output + assert "could not be read" in result.output + + def test_whoami_names_the_kill_switch_rather_than_a_missing_package( + self, isolated_home, secret_vault_factory + ): + """Every unreachable keychain used to be described as a locked one needing the keyring + package installed. Someone who set the kill switch has the package and an unlocked keychain, + so that advice sends them to fix two things that were never wrong.""" + _write_token_file(isolated_home, key=None) + vault = secret_vault_factory(available=False, failure=KeyringDisabled()) + + result = self.runner.invoke(whoami, obj={"base_url": "https://test.example.com", "secret_vault": vault}) + + assert DISABLE_KEYRING_ENV_VAR in result.output + assert "pip install" not in result.output + + def test_print_token_points_an_install_without_keyring_at_the_package( + self, isolated_home, secret_vault_factory + ): + _write_token_file(isolated_home, key=None) + vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) + obj = {"base_url": "https://test.example.com", "secret_vault": vault} + + result = self.runner.invoke(print_token, obj=obj) + + assert result.exit_code == 1 + assert "pip install 'litellm[cli]'" in result.output + assert DISABLE_KEYRING_ENV_VAR not in result.output class TestApiKeyPrecedence: From 26f62377451369bd1b96d2fbab11205c1018575c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 01:39:43 -0700 Subject: [PATCH 09/53] build: skip deleted files in the changed-file ruff format check `make lint` hands every path in the diff against the base branch to `ruff format --check`, including the ones the branch deleted, so any branch that moves or removes a file under `litellm/` fails the gate with "No such file or directory" instead of a formatting complaint. test-linting.yml already filters those out with `--diff-filter=ACMR`, so the Makefile was the half that drifted. Match it. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5e5f7c80027..ab6eba880bf 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ lint-install: # only the litellm Python files changed vs the base are checked, so a pre-existing # format issue elsewhere doesn't block an unrelated commit. lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) - @files=$$(git diff --name-only origin/litellm_internal_staging...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' || true); \ + @files=$$(git diff --name-only --diff-filter=ACMR origin/litellm_internal_staging...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' || true); \ if [ -z "$$files" ]; then \ echo "No changed litellm Python files to format-check."; \ else \ From ba637553f8cded70ddab429c429c9c030f29dcc5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 01:39:43 -0700 Subject: [PATCH 10/53] fix(cli): keep lite login and logout honest when the keychain will not answer Three ways the credential commands could mislead or hang. `lite logout` on a machine that never logged in warned that a credential may be stranded in a keychain it could not check, and told the user to install keyring to go clear it. There was nothing there. A missing token file is now read as the evidence it is, because logout keeps a secret-free file behind whenever the keychain is left unconfirmed, so a later run can tell a machine with a credential it cannot reach apart from one that never had a login. That holds on the LITELLM_CLI_DISABLE_KEYRING path too. `KeyringDiscardsWrites` was handled on the read and erase paths, which cannot produce it: the null backend returns None from `get_password` rather than raising, so only a write ever detects it. It now lives on `SecretWrite` alone and the unreachable arms are gone. `keyring.set_password` blocks forever under a HOME with no usable login keychain, which is what containers, CI images, `sudo -H`, and service accounts run with, and reads answer normally there so nothing cheaper tells them apart. `lite login` never touched a keychain before this, so a sign-in that simply never returns would be a new way for it to fail. Writes are pre-flighted with a throwaway value on a bounded wait, and a keychain that stays silent falls back to the token file. The real credential is never the thing handed to a call that might land long after we stopped waiting. Saving also stages the token file before the keychain is given anything, since the file is the half a read-only or full directory refuses. A save that cannot land now leaves both stores as it found them, which matters most when the login it failed to replace still works. --- litellm/litellm_core_utils/cli_keyring.py | 53 +++++++- litellm/litellm_core_utils/cli_token_utils.py | 88 +++++++----- litellm/proxy/client/cli/commands/auth.py | 7 +- pyproject.toml | 2 +- .../test_cli_token_utils.py | 125 +++++++++++++++++- .../proxy/client/cli/test_auth_commands.py | 2 +- 6 files changed, 227 insertions(+), 50 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index 15282fc522c..8da3e5226d4 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -11,18 +11,24 @@ token file and tell the user what to do about it. A write is only reported as stored once it has been read back, because keyring's null backend, which `keyring --disable` and headless CI images both select, -accepts every write and keeps nothing. +accepts every write and keeps nothing. Writes are also pre-flighted with a +throwaway value, because a keychain can answer neither way and block forever. """ import os +import threading +from contextlib import suppress from dataclasses import dataclass from typing import Final, Protocol, TypeAlias KEYRING_SERVICE: Final = "litellm-cli" KEYRING_ACCOUNT: Final = "credential" +KEYRING_PREFLIGHT_ACCOUNT: Final = "credential-preflight" DISABLE_KEYRING_ENV_VAR: Final = "LITELLM_CLI_DISABLE_KEYRING" _DISABLED_VALUES: Final = frozenset(("1", "true", "yes", "on")) +_PREFLIGHT_VALUE: Final = "preflight" +_PREFLIGHT_TIMEOUT_SECONDS: Final = 5.0 @dataclass(frozen=True, slots=True) @@ -70,9 +76,9 @@ class KeyringDiscardsWrites: pass -KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable | KeyringDiscardsWrites +KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable SecretRead: TypeAlias = SecretFound | SecretMissing | KeyringUnusable -SecretWrite: TypeAlias = SecretStored | KeyringUnusable +SecretWrite: TypeAlias = SecretStored | KeyringUnusable | KeyringDiscardsWrites SecretErase: TypeAlias = SecretErased | SecretStranded | KeyringUnusable @@ -113,10 +119,43 @@ def _keyring_api() -> KeyringApi | KeyringNotInstalled | KeyringDisabled: return KeyringNotInstalled() if api is None else api +def _answers_a_write(api: KeyringApi, timeout_seconds: float) -> bool: + """Whether the keychain answers a write at all, asked with a value worth nothing. + + macOS derives the login keychain from `$HOME`, and `set_password` against a HOME with no usable + one blocks forever with no timeout of its own. Containers, CI images, `sudo -H`, and service + accounts all run there, and reads answer normally, so nothing cheaper tells them apart. Asking + with a throwaway value keeps a keychain that never answers from taking `lite login` down with + it, and keeps the real credential out of a store that might accept it long after we gave up. + A keychain that refuses the probe outright still answered it, so only silence counts against it. + """ + answered: Final = threading.Event() + + def ask() -> None: + with suppress(Exception): + api.set_password(KEYRING_SERVICE, KEYRING_PREFLIGHT_ACCOUNT, _PREFLIGHT_VALUE) + answered.set() + + threading.Thread(target=ask, daemon=True, name="litellm-cli-keyring-preflight").start() + return answered.wait(timeout_seconds) + + +def _forget_the_preflight(api: KeyringApi) -> None: + """Take the throwaway probe back out. + + A backend that kept nothing has nothing to remove, and the probe is worth nothing either way, + so a keychain that refuses to give it up costs the caller nothing. + """ + with suppress(Exception): + api.delete_password(KEYRING_SERVICE, KEYRING_PREFLIGHT_ACCOUNT) + + @dataclass(frozen=True, slots=True) class KeyringVault: """The OS keychain, reached through the optional `keyring` package.""" + preflight_timeout_seconds: float = _PREFLIGHT_TIMEOUT_SECONDS + def read(self) -> SecretRead: api: Final = _keyring_api() if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): @@ -134,10 +173,16 @@ class KeyringVault: and `PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring` select, raises nothing to distinguish itself. Reading the value back is the only way to tell it apart from a keychain that really stored the credential, and the caller is about to drop its own copy on our word. + + The keychain is pre-flighted first, because one that blocks rather than answering would + otherwise hang `lite login` outright. """ api: Final = _keyring_api() if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): return api + if not _answers_a_write(api, self.preflight_timeout_seconds): + return KeyringUnreachable() + _forget_the_preflight(api) try: api.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, blob) except Exception: # noqa: BLE001 # a keychain that refuses the write falls back to the token file @@ -153,7 +198,7 @@ class KeyringVault: the caller knows whether this machine ever put a secret in a keychain. """ match self.read(): - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites() as unusable: + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() as unusable: return unusable case SecretMissing(): return SecretErased() diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index af1b918fc2a..825967e6866 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -23,7 +23,6 @@ from pydantic import BaseModel, ConfigDict, ValidationError from litellm.litellm_core_utils.cli_keyring import ( SYSTEM_KEYRING, KeyringDisabled, - KeyringDiscardsWrites, KeyringNotInstalled, KeyringUnreachable, SecretErase, @@ -53,6 +52,8 @@ class CredentialNotSaved: SecretSave: TypeAlias = SecretWrite | CredentialNotSaved +_UNREPLACEABLE_FILE: Final = "the staged file could not replace the one already there" + class CliTokenRecord(BaseModel): """A stored CLI credential. @@ -106,57 +107,71 @@ def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | N def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretSave: """Store a freshly minted credential. Reports where its secret material ended up, and why. - The token file is what makes a keychain-backed credential findable again, so a file that will - not be written takes the keychain copy down with it rather than leaving a live credential - stored under a machine that has no record of it. + The token file is what makes a keychain-backed credential findable again, and it is also the + half that a read-only or full directory refuses, so it is staged before the keychain is handed + anything. A save that cannot land then leaves both stores exactly as it found them, which + matters most when the login it failed to replace is still perfectly good. """ + staged: Final = _stage_token_file(_without_secret(record)) + if isinstance(staged, CredentialNotSaved): + return staged outcome: Final = ( SecretStored() if record.key is None else vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)) ) + if isinstance(outcome, SecretStored): + return outcome if _commit_token_file(staged) else CredentialNotSaved(_UNREPLACEABLE_FILE) + discard_staged_json(staged) + return _keep_the_secret_in_the_file(record, outcome) + + +def _keep_the_secret_in_the_file(record: CliTokenRecord, outcome: SecretWrite) -> SecretSave: + """Fall back to the owner-only file, which is all that is left when no keychain took the secret""" try: - _write_token_file(_without_secret(record) if isinstance(outcome, SecretStored) else record) + _write_token_file(record) except OSError as error: - if record.key is not None and isinstance(outcome, SecretStored): - vault.erase() return CredentialNotSaved(str(error)) return outcome def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretErase: - """Remove the credential from both stores. Reports whether the keychain is now free of it""" + """Remove the credential from both stores. Reports whether the keychain is now free of it. + + A file that holds no secret of its own is kept when the keychain will not confirm the entry is + gone, because it is the only remaining record that something is still in there to remove. That + is what lets a later run tell a machine with a credential it cannot reach apart from one that + never had a login at all. Anything still holding a secret is removed either way. + """ outcome: Final = vault.erase() - settled: Final = _nothing_left_behind(outcome) - Path(get_cli_token_file_path()).unlink(missing_ok=True) + record: Final = _read_token_file() + settled: Final = _nothing_left_behind(outcome, record) + if settled or record is None or record.key is not None: + Path(get_cli_token_file_path()).unlink(missing_ok=True) return SecretErased() if settled else outcome -def _nothing_left_behind(outcome: SecretErase) -> bool: +def _nothing_left_behind(outcome: SecretErase, record: CliTokenRecord | None) -> bool: """Whether the keychain can be trusted to hold no credential of ours once the file is gone. - A keychain that exists but is out of reach right now is never trusted, whatever the token file - looks like: the login that stored a secret there and the logout that cannot remove it are - separate runs, free to differ in whether the keychain was usable at the time. + A machine with no token file has no stored login to end, and `clear_cli_token` keeps one behind + whenever the keychain is left unconfirmed, so a missing file is real evidence rather than the + absence of it. Past that, a keychain that exists but is out of reach right now is + never trusted, whatever the file looks like: the login that stored a secret there and the + logout that cannot remove it are separate runs, free to differ in whether the keychain was + usable at the time. The exception is a missing `keyring` package, which had to be missing when + the credential was stored too, so a file still holding its own secret proves no keychain was + ever involved. `SecretStranded` is the keychain answering for itself and outranks the file. """ match outcome: case SecretErased(): return True - case SecretStranded() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites(): + case SecretStranded(): return False + case KeyringDisabled() | KeyringUnreachable(): + return record is None case KeyringNotInstalled(): - return _file_holds_its_own_secret() - - -def _file_holds_its_own_secret() -> bool: - """Whether the stored login keeps its secret in the token file, ruling out a keychain entry. - - Sound only against a missing `keyring` package, the one way to lose the keychain that had to - hold at storage time too, since nothing here can reach a keychain without it. A file whose - secret half is absent went to a keychain by definition, and so rules nothing out. - """ - record: Final = _read_token_file() - return record is not None and record.key is not None + return record is None or record.key is not None def get_litellm_gateway_api_key( @@ -227,7 +242,7 @@ def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecor return _apply_vault_secret(record, blob, vault) case SecretMissing(): return _migrate_file_secret(record, vault) - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites(): + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): return record @@ -265,7 +280,7 @@ def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliToken if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): discard_staged_json(staged) return record - if not _commit_scrubbed_file(staged): + if not _commit_token_file(staged): vault.erase() return record @@ -275,19 +290,24 @@ def _scrub_file_secret(record: CliTokenRecord) -> bool: if record.key is None and not record.jwt_token: return True staged: Final = _stage_scrubbed_file(record) - return staged is not None and _commit_scrubbed_file(staged) + return staged is not None and _commit_token_file(staged) def _stage_scrubbed_file(record: CliTokenRecord) -> str | None: + staged: Final = _stage_token_file(_without_secret(record)) + return None if isinstance(staged, CredentialNotSaved) else staged + + +def _stage_token_file(record: CliTokenRecord) -> str | CredentialNotSaved: path: Final = Path(get_cli_token_file_path()) try: ensure_private_dir(path.parent) - return stage_private_json(str(path), _without_secret(record).model_dump(exclude_none=True)) - except OSError: - return None + return stage_private_json(str(path), record.model_dump(exclude_none=True)) + except OSError as error: + return CredentialNotSaved(str(error)) -def _commit_scrubbed_file(staged: str) -> bool: +def _commit_token_file(staged: str) -> bool: try: commit_staged_json(staged, get_cli_token_file_path()) except OSError: diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index eba9994f7ec..d89641d2366 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -123,7 +123,8 @@ def storage_notice(outcome: SecretSave) -> str: case CredentialNotSaved(detail=detail): return ( f"Signed in, but the credential could not be saved to {path}: {detail}. " - "Nothing was kept, so run 'lite login' again once that path is writable." + "Any login you already had is untouched. Run 'lite login' again once that path is " + "writable, or 'lite logout' to clear whatever is stored now." ) @@ -140,7 +141,7 @@ def keychain_unreadable_notice(vault: SecretVault) -> str: f"Your credential is in your OS keychain, which {DISABLE_KEYRING_ENV_VAR} is blocking. " "Unset it, or run 'lite login' to start over." ) - case KeyringUnreachable() | KeyringDiscardsWrites(): + case KeyringUnreachable(): return ( "Your credential is in your OS keychain, which could not be read. Unlock it, or run " "'lite login' to start over." @@ -800,7 +801,7 @@ def logout(ctx: click.Context): case KeyringDisabled(): click.echo(UNCHECKED_KEYCHAIN_MESSAGE) click.echo(f"Unset {DISABLE_KEYRING_ENV_VAR} and run 'lite logout' again to clear it.") - case KeyringUnreachable() | KeyringDiscardsWrites(): + case KeyringUnreachable(): click.echo(UNCHECKED_KEYCHAIN_MESSAGE) click.echo("Unlock your keychain and run 'lite logout' again to clear it.") diff --git a/pyproject.toml b/pyproject.toml index 32921e14d31..64adb9cd595 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ proxy = [ ] # Thin client install for the `lite` CLI on developer laptops. The CLI's heavy # imports (fastapi, cryptography, ...) are all guarded, so it runs on the base -# SDK plus just these four; none of the server runtime in `proxy` is pulled in. +# SDK plus just these five; none of the server runtime in `proxy` is pulled in. cli = [ "rich>=13.9.4,<14.0", "pyyaml>=6.0.3,<7.0", diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index e0f5f99dc1b..69ce47b25b3 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -2,6 +2,7 @@ import json import os import stat import sys +import threading import time import pytest @@ -10,6 +11,7 @@ from litellm.constants import CLI_JWT_EXPIRATION_HOURS from litellm.litellm_core_utils.cli_keyring import ( DISABLE_KEYRING_ENV_VAR, KEYRING_ACCOUNT, + KEYRING_PREFLIGHT_ACCOUNT, KEYRING_SERVICE, KeyringVault, SecretFound, @@ -329,12 +331,12 @@ class TestSaveCliToken: assert isinstance(outcome, CredentialNotSaved) assert "read-only file system" in outcome.detail - def test_a_credential_the_file_will_not_record_is_taken_back_out_of_the_keychain( + def test_a_file_that_will_not_be_written_stops_the_save_before_the_keychain_is_touched( self, isolated_home, secret_vault_factory, monkeypatch ): - """The token file is what makes a keychain entry findable again. Leaving the secret in the - keychain with nothing pointing at it strands a live credential under a machine that has no - idea it is there, and no `lite logout` would ever go looking for it.""" + """The token file is what makes a keychain entry findable again, so it is staged first. + Handing the keychain a secret and only then finding out that nothing will point at it + would strand a live credential under a machine with no idea it is there.""" vault = secret_vault_factory() def _explode(*args, **kwargs): @@ -346,6 +348,26 @@ class TestSaveCliToken: assert vault.blob is None + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") + def test_a_login_that_cannot_be_saved_leaves_the_working_one_alone( + self, isolated_home, secret_vault_factory + ): + """Signing in again on a machine whose ~/.litellm has gone read-only must not cost the user + the credential they already had. Overwriting the keychain and then failing to record it, or + undoing that write afterwards, would take a login that still works out from under them.""" + _write_legacy_file(isolated_home, key=None) + vault = secret_vault_factory(blob=_blob(key="sk-in-use")) + path = _token_file(isolated_home) + path.parent.chmod(0o500) + try: + outcome = save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) + finally: + path.parent.chmod(0o700) + + assert isinstance(outcome, CredentialNotSaved) + assert json.loads(vault.blob)["key"] == "sk-in-use" + assert load_cli_token(vault=vault).key == "sk-in-use" + def test_a_failed_write_leaves_the_previous_credential_intact(self, isolated_home, secret_vault_factory, monkeypatch): path = _write_legacy_file(isolated_home) before = path.read_text() @@ -460,8 +482,44 @@ class TestClearCliToken: vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) assert clear_cli_token(vault=vault) == KeyringNotInstalled() + assert json.loads(_token_file(isolated_home).read_text()).get("key") is None + + def test_a_logout_that_cannot_clear_the_keychain_keeps_the_record_that_it_has_to( + self, isolated_home, secret_vault_factory + ): + """The file left behind holds no secret. It is what a later run reads to tell a machine with + a credential it cannot reach apart from one that never had a login, which is the difference + between warning the user and inventing a credential for them to worry about.""" + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(available=False, failure=KeyringUnreachable()) + + clear_cli_token(vault=vault) + + assert json.loads(_token_file(isolated_home).read_text()).get("key") is None + + def test_a_logout_that_cannot_clear_the_keychain_still_takes_the_file_secret_away( + self, isolated_home, secret_vault_factory + ): + """Keeping a record of the unreachable keychain must never mean keeping the cleartext copy + the user just asked to be rid of.""" + _write_legacy_file(isolated_home) + vault = secret_vault_factory(available=False, failure=KeyringUnreachable()) + + clear_cli_token(vault=vault) + assert not _token_file(isolated_home).exists() + @pytest.mark.parametrize("failure", [KeyringNotInstalled(), KeyringDisabled(), KeyringUnreachable()]) + def test_logging_out_of_a_machine_that_never_logged_in_invents_nothing_to_warn_about( + self, isolated_home, secret_vault_factory, failure + ): + """`lite logout` with no token file has nothing to end. Warning that a credential may be + stranded in a keychain it cannot check sends the user after something that was never there, + and `pip install keyring` will not make it appear.""" + vault = secret_vault_factory(available=False, failure=failure) + + assert clear_cli_token(vault=vault) == SecretErased() + def test_a_file_backed_login_logs_out_quietly_without_keyring(self, isolated_home, secret_vault_factory): """The complement, and the one inference the file does support: nothing here can reach a keychain without the package, so an install that lacks it and a file that still holds its @@ -510,7 +568,7 @@ class _FakeKeyringModule: self.calls.append(("set", service_name, username)) if self.set_error is not None: raise self.set_error - if self.discard: + if self.discard or username != KEYRING_ACCOUNT: return self.stored = password @@ -518,7 +576,22 @@ class _FakeKeyringModule: self.calls.append(("delete", service_name, username)) if self.delete_error is not None: raise self.delete_error - self.stored = None + if username == KEYRING_ACCOUNT: + self.stored = None + + +class _NeverAnsweringKeyringModule(_FakeKeyringModule): + """A keychain whose writes block instead of returning, the way macOS does under a HOME that + has no usable login keychain.""" + + def __init__(self): + super().__init__() + self.blocked = threading.Event() + + def set_password(self, service_name, username, password): + self.calls.append(("set", service_name, username)) + self.blocked.set() + threading.Event().wait() @pytest.fixture @@ -540,7 +613,8 @@ class TestKeyringVault: assert vault.read() == SecretFound("blob-1") assert vault.erase() == SecretErased() assert vault.read() == SecretMissing() - assert {call[1:] for call in fake.calls} == {(KEYRING_SERVICE, KEYRING_ACCOUNT)} + assert {call[1] for call in fake.calls} == {KEYRING_SERVICE} + assert {call[2] for call in fake.calls} == {KEYRING_ACCOUNT, KEYRING_PREFLIGHT_ACCOUNT} def test_the_kill_switch_reports_no_keychain(self, monkeypatch): """`LITELLM_CLI_DISABLE_KEYRING` has to work without importing keyring, because keyring @@ -590,6 +664,43 @@ class TestKeyringVault: assert KeyringVault().write("blob-1") == KeyringDiscardsWrites() assert fake.stored is None + def test_a_keychain_that_never_answers_does_not_hang_the_login(self, install_fake_keyring): + """macOS derives the login keychain from `$HOME`, and `set_password` under a HOME with no + usable one blocks forever with no timeout of its own. Containers, CI images, `sudo -H`, and + service accounts all run there, and `lite login` never touched a keychain before this, so a + sign-in that simply never returns would be a new way for it to fail.""" + fake = install_fake_keyring(_NeverAnsweringKeyringModule()) + vault = KeyringVault(preflight_timeout_seconds=0.2) + + started = time.monotonic() + outcome = vault.write("blob-1") + + assert outcome == KeyringUnreachable() + assert time.monotonic() - started < 5 + assert fake.blocked.is_set() + + def test_a_keychain_that_never_answers_is_never_handed_the_credential(self, install_fake_keyring): + """Giving up on the write is only safe if the secret was never the thing being written. A + blocked call can still land later, and a keychain copy nobody waited for would sit beside + the file copy the user was told about.""" + fake = install_fake_keyring(_NeverAnsweringKeyringModule()) + + KeyringVault(preflight_timeout_seconds=0.2).write("blob-1") + + assert [call[2] for call in fake.calls] == [KEYRING_PREFLIGHT_ACCOUNT] + + def test_a_login_survives_a_keychain_that_never_answers(self, isolated_home, install_fake_keyring): + """The end of the same story: the credential still has to be usable afterwards.""" + install_fake_keyring(_NeverAnsweringKeyringModule()) + + outcome = save_cli_token( + CliTokenRecord(base_url=SERVER, key="sk-only-copy"), + vault=KeyringVault(preflight_timeout_seconds=0.2), + ) + + assert outcome == KeyringUnreachable() + assert json.loads(_token_file(isolated_home).read_text())["key"] == "sk-only-copy" + def test_the_real_null_backend_is_rejected(self, monkeypatch): """Pinned against the actual library rather than the double above, because the whole risk is that upstream's no-op write looks exactly like a successful one.""" diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 8e1551c0720..e491dbd5aca 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -1074,7 +1074,7 @@ class TestKeychainBackedCommands: assert result.exit_code == 0 assert "could not be removed" in result.output - assert not (isolated_home / ".litellm" / "token.json").exists() + assert json.loads((isolated_home / ".litellm" / "token.json").read_text()).get("key") is None def test_print_token_explains_a_locked_keychain_instead_of_printing_nothing( self, isolated_home, secret_vault_factory From c7da91d47f95aa9aa992264c1979748b916de796 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 01:47:50 -0700 Subject: [PATCH 11/53] fix(cli): say so when the keychain took a credential the file cannot name Staging the token file can succeed and the replacement still fail afterwards, and that is the one save path where the keychain has already taken the new secret. It was reported as a save that kept nothing, which sends the user looking for a credential that is sitting in their keychain, and it claimed the previous login was untouched when the one keychain slot had just been written over. Give that path its own outcome and its own notice. The new secret stays where it is: the entry it replaced went the moment it landed, so no rollback brings that back, and removing the new one too would turn a login this machine may still be able to use into no login at all. The remaining `CredentialNotSaved` paths all leave both stores untouched, so the reassurance they carry is now true wherever it is printed. --- litellm/litellm_core_utils/cli_token_utils.py | 23 ++++++++++--- litellm/proxy/client/cli/commands/auth.py | 9 +++++- .../test_cli_token_utils.py | 32 +++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 825967e6866..15b1390f337 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -45,14 +45,25 @@ from litellm.litellm_core_utils.private_json import ( @dataclass(frozen=True, slots=True) class CredentialNotSaved: - """The credential was minted but no store would keep it, so this machine has none.""" + """The credential was minted but no store would keep it, so this machine has none. + + Nothing was touched on the way to this, so a login that already worked still does. + """ detail: str -SecretSave: TypeAlias = SecretWrite | CredentialNotSaved +@dataclass(frozen=True, slots=True) +class CredentialNotRecorded: + """The keychain took the credential, but the file that names it could not be replaced. -_UNREPLACEABLE_FILE: Final = "the staged file could not replace the one already there" + The keychain holds one entry, so the secret that was there is already gone and no rollback + brings it back. Removing the new one as well would only turn a login this machine may still + be able to use into no login at all, so it stays, and the user is told what is where. + """ + + +SecretSave: TypeAlias = SecretWrite | CredentialNotSaved | CredentialNotRecorded class CliTokenRecord(BaseModel): @@ -111,6 +122,10 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN half that a read-only or full directory refuses, so it is staged before the keychain is handed anything. A save that cannot land then leaves both stores exactly as it found them, which matters most when the login it failed to replace is still perfectly good. + + Staging can still succeed and the replacement fail afterwards. That is the one case where the + keychain has already taken the new secret, and it reports itself as such rather than claiming + the previous login survived. """ staged: Final = _stage_token_file(_without_secret(record)) if isinstance(staged, CredentialNotSaved): @@ -121,7 +136,7 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN else vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)) ) if isinstance(outcome, SecretStored): - return outcome if _commit_token_file(staged) else CredentialNotSaved(_UNREPLACEABLE_FILE) + return outcome if _commit_token_file(staged) else CredentialNotRecorded() discard_staged_json(staged) return _keep_the_secret_in_the_file(record, outcome) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index d89641d2366..eb956cd536c 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.cli_keyring import ( ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, + CredentialNotRecorded, CredentialNotSaved, SecretSave, clear_cli_token, @@ -126,6 +127,12 @@ def storage_notice(outcome: SecretSave) -> str: "Any login you already had is untouched. Run 'lite login' again once that path is " "writable, or 'lite logout' to clear whatever is stored now." ) + case CredentialNotRecorded(): + return ( + f"Signed in, and the credential is in your OS keychain, but {path} could not be " + "replaced, so this machine may still be using your previous login. Run 'lite login' " + "again once that path is writable, or 'lite logout' to clear both." + ) def keychain_unreadable_notice(vault: SecretVault) -> str: @@ -754,7 +761,7 @@ def login(ctx: click.Context, config_claude: bool): click.echo("\nLogin successful!") click.echo(f"JWT Token: {api_key[:20]}...") click.echo(storage_notice(stored)) - if isinstance(stored, CredentialNotSaved): + if isinstance(stored, (CredentialNotSaved, CredentialNotRecorded)): return click.echo("You can now use the CLI without specifying --api-key") diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 69ce47b25b3..0cf1e55d363 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -26,6 +26,7 @@ from litellm.litellm_core_utils.cli_keyring import ( ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, + CredentialNotRecorded, CredentialNotSaved, clear_cli_token, get_cli_token_file_path, @@ -368,6 +369,37 @@ class TestSaveCliToken: assert json.loads(vault.blob)["key"] == "sk-in-use" assert load_cli_token(vault=vault).key == "sk-in-use" + def test_a_keychain_write_the_file_cannot_be_pointed_at_is_reported_as_that( + self, isolated_home, secret_vault_factory + ): + """Staging the file can succeed and the replacement still fail, and that is the one path + where the keychain already took the new secret. Reporting it as a save that kept nothing + would send the user looking for a credential that is sitting in their keychain.""" + vault = secret_vault_factory() + path = _token_file(isolated_home) + path.parent.mkdir(parents=True, exist_ok=True) + path.mkdir() + + outcome = save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) + + assert isinstance(outcome, CredentialNotRecorded) + assert json.loads(vault.blob)["key"] == "sk-new" + + def test_the_credential_the_file_cannot_name_is_left_in_the_keychain( + self, isolated_home, secret_vault_factory + ): + """The keychain holds one entry, so the secret that was there went the moment this one + landed. Taking the new one back out would turn a login this machine may still be able to + use into no login at all, and it cannot restore the old one either way.""" + vault = secret_vault_factory(blob=_blob(key="sk-in-use")) + path = _token_file(isolated_home) + path.parent.mkdir(parents=True, exist_ok=True) + path.mkdir() + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) + + assert vault.blob is not None + def test_a_failed_write_leaves_the_previous_credential_intact(self, isolated_home, secret_vault_factory, monkeypatch): path = _write_legacy_file(isolated_home) before = path.read_text() From ef6af5c615815592da85e26315b6212046ba82ba Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:03:27 -0700 Subject: [PATCH 12/53] test(e2e): accept both model-not-found phrasings on a shared proxy /audio/transcriptions answers a model-less request with one of two 400s depending on whether any wildcard deployment is registered at the time, and every suite shares one proxy, so run order decided which message came back. The assertion pinned only the no-wildcard wording, so it went red whenever the model-access-group suite had registered its wildcards first. It now accepts either message and still holds the error to naming the model Verified against a live proxy in both states: with a wildcard registered (the message CI was seeing) and with none (the message the assertion expected), the suite passes 3/3 either way --- .../llm_translation/test_audio_transcriptions_e2e.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py index 92b33fef85f..735f1a4a703 100644 --- a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py +++ b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py @@ -3,7 +3,10 @@ Registers an OpenAI speech-to-text deployment at runtime and uploads a spoken weather question (the realtime suite's 24kHz WAV fixture) as multipart, asserting the returned transcript is non-empty and mentions the word it was asked about. -Also pins missing file/model negatives. +Also pins missing file/model negatives. A model-less request comes back as one of +two 400s depending on whether any wildcard deployment happens to be registered on +the shared proxy, so the assertion accepts either phrasing and holds both to naming +the model as the problem. """ from __future__ import annotations @@ -25,6 +28,8 @@ WEATHER_WAV = ( Path(__file__).resolve().parent / "realtime" / "fixtures" / "weather_question_24k.wav" ) +MISSING_MODEL_PHRASES: Final = ("model=none", "invalid model", "model is required") + class _OptionalTranscriptionForm(BaseModel): model: str | None = None @@ -105,8 +110,8 @@ class TestAudioTranscriptions: match result: case UnknownApiError(status_code=400, body=body): lowered: Final = body.lower() - assert "model" in lowered and ("required" in lowered or "invalid model" in lowered), ( - f"missing model error must identify the required model: {body[:300]}" + assert any(phrase in lowered for phrase in MISSING_MODEL_PHRASES), ( + f"missing model error must name the model as the problem: {body[:300]}" ) case other: pytest.fail(f"missing model expected a model-specific 400, got {other!r}") From b6fef179ff151e2cb88990ed24fe2613a1824704 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:08:04 -0700 Subject: [PATCH 13/53] fix(cli): stop a repeat logout from retracting its own keychain warning A logout that could not reach the keychain deleted the token file whenever it still held its own secret, and the next logout read that missing file as proof the keychain was clean. It answered the warning the first run had just issued with "Logged out successfully" while the entry an earlier login left behind was still live. The file is the only record that something may still be in there, which is what `_nothing_left_behind` already says it relies on, so keep it and take only the secret out. A keychain that did answer is a different case. `SecretStranded` means the entry is confirmed there and would not delete, and that needs no note in the file, while keeping one lets every later command read the credential straight back out of the keychain, which makes "Logged out locally" untrue. That one drops the file, as it did before. The secret still goes first either way: a copy that cannot be replaced with a secret-free one is removed rather than kept. --- litellm/litellm_core_utils/cli_token_utils.py | 24 ++++++++++--- .../test_cli_token_utils.py | 34 +++++++++++++++++-- .../proxy/client/cli/test_auth_commands.py | 2 +- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 15b1390f337..4e01dd723ce 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -153,19 +153,33 @@ def _keep_the_secret_in_the_file(record: CliTokenRecord, outcome: SecretWrite) - def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretErase: """Remove the credential from both stores. Reports whether the keychain is now free of it. - A file that holds no secret of its own is kept when the keychain will not confirm the entry is - gone, because it is the only remaining record that something is still in there to remove. That - is what lets a later run tell a machine with a credential it cannot reach apart from one that - never had a login at all. Anything still holding a secret is removed either way. + A logout the keychain never answered keeps the token file, with its secret taken out, because + that file is the only remaining record that something may still be in there to remove. It is + what lets a later run tell a machine with a credential it cannot reach apart from one that never + had a login at all, and taking it away would leave the next logout answering the warning this + one just issued with a false all-clear. The secret goes either way. """ outcome: Final = vault.erase() record: Final = _read_token_file() settled: Final = _nothing_left_behind(outcome, record) - if settled or record is None or record.key is not None: + if settled or not _keep_the_unchecked_keychain_on_record(outcome, record): Path(get_cli_token_file_path()).unlink(missing_ok=True) return SecretErased() if settled else outcome +def _keep_the_unchecked_keychain_on_record(outcome: SecretErase, record: CliTokenRecord | None) -> bool: + """Whether the token file, stripped of its secret, is worth keeping as the note that says so. + + Only a keychain that could not be reached leaves the question open. One that answered for itself + is remembered without any help from the file, and a file it can still pair a live entry with + would leave the machine signed in to the login that was just ended. A copy that cannot be + replaced with a secret-free one is not kept either, because the secret goes first. + """ + if record is None or isinstance(outcome, SecretStranded): + return False + return _scrub_file_secret(record) + + def _nothing_left_behind(outcome: SecretErase, record: CliTokenRecord | None) -> bool: """Whether the keychain can be trusted to hold no credential of ours once the file is gone. diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 0cf1e55d363..162e5dd4b67 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -477,6 +477,18 @@ class TestClearCliToken: assert clear_cli_token(vault=vault) == SecretStranded() assert not _token_file(isolated_home).exists() + def test_a_keychain_that_will_not_release_the_secret_still_ends_the_local_login( + self, isolated_home, secret_vault_factory + ): + """The warning this returns says the machine is logged out locally and the keychain entry is + what is left over. Keeping the file that names that entry makes the first half untrue: every + later command reads the credential straight back out of the keychain and keeps working.""" + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(blob=_blob(), erasable=False) + + assert clear_cli_token(vault=vault) == SecretStranded() + assert load_cli_token(vault=vault) is None + @pytest.mark.parametrize( "failure", [KeyringDisabled(), KeyringUnreachable(), KeyringDiscardsWrites()] ) @@ -491,7 +503,7 @@ class TestClearCliToken: vault = secret_vault_factory(available=False, failure=failure) assert clear_cli_token(vault=vault) == failure - assert not _token_file(isolated_home).exists() + assert json.loads(_token_file(isolated_home).read_text()).get("key") is None def test_a_second_logout_still_reports_the_keychain_it_could_not_clear( self, isolated_home, secret_vault_factory @@ -539,7 +551,25 @@ class TestClearCliToken: clear_cli_token(vault=vault) - assert not _token_file(isolated_home).exists() + assert "sk-legacy" not in _token_file(isolated_home).read_text() + + def test_a_repeat_logout_never_answers_its_own_warning_with_an_all_clear( + self, isolated_home, secret_vault_factory + ): + """Sign in while the keychain works, sign in again once it has gone out of reach so the + second secret lands in the file, then log out twice. The first logout cannot say the first + login's entry is gone, and says so. If the second one reads the file the first one took + away as proof of a clean keychain, it retracts that warning while the credential behind it + is still live.""" + vault = secret_vault_factory() + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-first"), vault=vault) + vault.available = False + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-second"), vault=vault) + + assert clear_cli_token(vault=vault) == KeyringUnreachable() + assert clear_cli_token(vault=vault) == KeyringUnreachable() + assert vault.blob is not None + assert "sk-second" not in _token_file(isolated_home).read_text() @pytest.mark.parametrize("failure", [KeyringNotInstalled(), KeyringDisabled(), KeyringUnreachable()]) def test_logging_out_of_a_machine_that_never_logged_in_invents_nothing_to_warn_about( diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index e491dbd5aca..8e1551c0720 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -1074,7 +1074,7 @@ class TestKeychainBackedCommands: assert result.exit_code == 0 assert "could not be removed" in result.output - assert json.loads((isolated_home / ".litellm" / "token.json").read_text()).get("key") is None + assert not (isolated_home / ".litellm" / "token.json").exists() def test_print_token_explains_a_locked_keychain_instead_of_printing_nothing( self, isolated_home, secret_vault_factory From f86aeba1e7b1ed1395a5b6d5b5c47c2e6c94d7ca Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:21:21 -0700 Subject: [PATCH 14/53] fix(cli): stop a file-held secret from vouching for an unreadable keychain A logout run from an install without the keyring package treated a token file holding its own secret as proof that no keychain entry could exist. That only holds for the login which wrote the file. A login before it may have had the package and put its credential in the keychain, where it outlives both the uninstall and the file that replaced it, so logout reported a clean sweep over a live credential. Every keychain that cannot be reached is now treated the same way, and the message says the keychain went unchecked rather than asserting what is in it. --- litellm/litellm_core_utils/cli_token_utils.py | 15 +++++------- litellm/proxy/client/cli/commands/auth.py | 2 +- .../test_cli_token_utils.py | 23 ++++++++++++------- .../proxy/client/cli/test_auth_commands.py | 16 ++++++++----- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 4e01dd723ce..78b52f33e2d 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -185,22 +185,19 @@ def _nothing_left_behind(outcome: SecretErase, record: CliTokenRecord | None) -> A machine with no token file has no stored login to end, and `clear_cli_token` keeps one behind whenever the keychain is left unconfirmed, so a missing file is real evidence rather than the - absence of it. Past that, a keychain that exists but is out of reach right now is - never trusted, whatever the file looks like: the login that stored a secret there and the - logout that cannot remove it are separate runs, free to differ in whether the keychain was - usable at the time. The exception is a missing `keyring` package, which had to be missing when - the credential was stored too, so a file still holding its own secret proves no keychain was - ever involved. `SecretStranded` is the keychain answering for itself and outranks the file. + absence of it. Past that, a keychain that could not be reached is never trusted, whatever the + file looks like. Even a file holding its own secret says only that the login which wrote it had + no keychain to write to, and the login before it may well have had one: the entry that login + left outlives both the uninstalled package and the file that replaced it. `SecretStranded` is + the keychain answering for itself and outranks the file. """ match outcome: case SecretErased(): return True case SecretStranded(): return False - case KeyringDisabled() | KeyringUnreachable(): + case KeyringDisabled() | KeyringNotInstalled() | KeyringUnreachable(): return record is None - case KeyringNotInstalled(): - return record is None or record.key is not None def get_litellm_gateway_api_key( diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index eb956cd536c..5fcc53e69eb 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -803,7 +803,7 @@ def logout(ctx: click.Context): click.echo(STRANDED_CREDENTIAL_MESSAGE) click.echo("Unlock your keychain and run 'lite logout' again to clear it.") case KeyringNotInstalled(): - click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo(UNCHECKED_KEYCHAIN_MESSAGE) click.echo(f"Install the keyring package with: {KEYRING_INSTALL_HINT}, then run 'lite logout' again.") case KeyringDisabled(): click.echo(UNCHECKED_KEYCHAIN_MESSAGE) diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 162e5dd4b67..143c9e738a9 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -582,15 +582,22 @@ class TestClearCliToken: assert clear_cli_token(vault=vault) == SecretErased() - def test_a_file_backed_login_logs_out_quietly_without_keyring(self, isolated_home, secret_vault_factory): - """The complement, and the one inference the file does support: nothing here can reach a - keychain without the package, so an install that lacks it and a file that still holds its - own secret between them account for the whole credential.""" - _write_legacy_file(isolated_home) - vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) + def test_a_file_backed_login_cannot_vouch_for_a_keychain_no_package_can_reach( + self, isolated_home, secret_vault_factory + ): + """Sign in with the keyring package installed, lose the package, then sign in again so the + second secret lands in the file. The first login's entry outlives both, and the file that + replaced it holds a secret of its own, which is the shape a logout must not read as proof + that no keychain was ever involved.""" + vault = secret_vault_factory() + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-keychain"), vault=vault) + vault.available = False + vault.failure = KeyringNotInstalled() + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-in-file"), vault=vault) - assert clear_cli_token(vault=vault) == SecretErased() - assert not _token_file(isolated_home).exists() + assert clear_cli_token(vault=vault) == KeyringNotInstalled() + assert vault.blob is not None + assert "sk-in-file" not in _token_file(isolated_home).read_text() def test_is_safe_when_nothing_was_ever_stored(self, isolated_home, secret_vault_factory): assert clear_cli_token(vault=secret_vault_factory()) == SecretErased() diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 8e1551c0720..9e4ede88337 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -459,7 +459,7 @@ class TestLogoutCommand: assert result.exit_code == 0 assert "Logged out successfully" not in result.output - assert "still in the OS keychain" in result.output + assert "could not be checked" in result.output assert "pip install 'litellm[cli]'" in result.output def test_logout_does_not_call_an_unusable_keychain_clean(self, isolated_home, secret_vault_factory): @@ -492,9 +492,12 @@ class TestLogoutCommand: assert "still in the OS keychain" in result.output assert "Unlock your keychain" in result.output - def test_logout_from_a_file_only_login_stays_quiet(self, isolated_home, secret_vault_factory): - """The credential never went to a keychain, so removing the file is the whole logout and - warning about a keychain entry would send the user chasing one that cannot exist.""" + def test_logout_without_the_keyring_package_still_warns_about_a_file_held_secret( + self, isolated_home, secret_vault_factory + ): + """A file holding its own secret only says the login that wrote it had no keychain to write + to. An earlier login on this machine may have had one, and no install without the package + can look, so the honest answer is that the keychain went unchecked.""" _write_token_file(isolated_home, key="sk-in-file") result = self.runner.invoke( @@ -502,8 +505,9 @@ class TestLogoutCommand: ) assert result.exit_code == 0 - assert "Logged out successfully" in result.output - assert "still in the OS keychain" not in result.output + assert "Logged out successfully" not in result.output + assert "could not be checked" in result.output + assert "pip install 'litellm[cli]'" in result.output class TestWhoamiCommand: From ef104acdaf9ec791f1b1674fd7d6f7eabadeb6fc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:28:48 -0700 Subject: [PATCH 15/53] fix(cli): report a token file logout cannot remove instead of crashing A ~/.litellm that has gone read-only, or one left root-owned by a sudo login, refuses both the scrubbed rewrite and the removal. The removal was unguarded, so 'lite logout' ended in a PermissionError traceback with the credential still readable in the file. It now comes back as an outcome the command reports, naming the file and what to do about it, and a file that holds no secret is still not worth alarming anyone over. --- litellm/litellm_core_utils/cli_token_utils.py | 34 ++++++- litellm/proxy/client/cli/commands/auth.py | 5 + .../test_cli_token_utils.py | 91 +++++++++++++++++++ .../proxy/client/cli/test_auth_commands.py | 17 ++++ 4 files changed, 143 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 78b52f33e2d..f2c0f8ed001 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -63,8 +63,22 @@ class CredentialNotRecorded: """ +@dataclass(frozen=True, slots=True) +class CredentialNotCleared: + """The token file still holds the secret, because it could not be removed or rewritten. + + Logging out of the keychain is only half of it. A `~/.litellm` that refuses both the scrubbed + rewrite and the removal leaves the credential readable on disk, which is the one thing a logout + is for, so it is reported instead of being counted as a clean sweep. + """ + + detail: str + + SecretSave: TypeAlias = SecretWrite | CredentialNotSaved | CredentialNotRecorded +SecretClear: TypeAlias = SecretErase | CredentialNotCleared + class CliTokenRecord(BaseModel): """A stored CLI credential. @@ -150,23 +164,35 @@ def _keep_the_secret_in_the_file(record: CliTokenRecord, outcome: SecretWrite) - return outcome -def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretErase: +def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretClear: """Remove the credential from both stores. Reports whether the keychain is now free of it. A logout the keychain never answered keeps the token file, with its secret taken out, because that file is the only remaining record that something may still be in there to remove. It is what lets a later run tell a machine with a credential it cannot reach apart from one that never had a login at all, and taking it away would leave the next logout answering the warning this - one just issued with a false all-clear. The secret goes either way. + one just issued with a false all-clear. The secret goes either way, and a file that will give up + neither its copy nor itself outranks whatever the keychain had to say. """ outcome: Final = vault.erase() record: Final = _read_token_file() settled: Final = _nothing_left_behind(outcome, record) - if settled or not _keep_the_unchecked_keychain_on_record(outcome, record): - Path(get_cli_token_file_path()).unlink(missing_ok=True) + if not settled and _keep_the_unchecked_keychain_on_record(outcome, record): + return outcome + removal: Final = _remove_token_file() + if removal is not None and record is not None and record.key is not None: + return removal return SecretErased() if settled else outcome +def _remove_token_file() -> CredentialNotCleared | None: + try: + Path(get_cli_token_file_path()).unlink(missing_ok=True) + except OSError as error: + return CredentialNotCleared(str(error)) + return None + + def _keep_the_unchecked_keychain_on_record(outcome: SecretErase, record: CliTokenRecord | None) -> bool: """Whether the token file, stripped of its secret, is worth keeping as the note that says so. diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 5fcc53e69eb..b3a7db4bed4 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.cli_keyring import ( ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, + CredentialNotCleared, CredentialNotRecorded, CredentialNotSaved, SecretSave, @@ -796,9 +797,13 @@ def login(ctx: click.Context, config_claude: bool): @click.pass_context def logout(ctx: click.Context): """Logout and clear stored authentication""" + path: Final = get_cli_token_file_path() match clear_cli_token(vault=context_secret_vault(ctx)): case SecretErased(): click.echo("Logged out successfully. Authentication token cleared.") + case CredentialNotCleared(detail=detail): + click.echo(f"Your credential is still in {path}, which could not be removed: {detail}.") + click.echo("Delete that file, or make the directory writable and run 'lite logout' again.") case SecretStranded(): click.echo(STRANDED_CREDENTIAL_MESSAGE) click.echo("Unlock your keychain and run 'lite logout' again to clear it.") diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 143c9e738a9..2830fad58b4 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.cli_keyring import ( from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, CredentialNotRecorded, + CredentialNotCleared, CredentialNotSaved, clear_cli_token, get_cli_token_file_path, @@ -81,6 +82,26 @@ def _blob(base_url=SERVER, key="sk-vault", jwt_token=""): return json.dumps({"base_url": base_url, "key": key, "jwt_token": jwt_token}) +_REAL_REPLACE = os.replace + + +def _refuse_replace(*args, **kwargs): + raise OSError("device or resource busy") + + +class _ReplaceThatStartsRefusing: + """`os.replace` standing in for a path that cannot be replaced yet: a file another process holds + open on Windows, a directory that went read-only between staging and the rewrite.""" + + def __init__(self): + self.allowed = False + + def __call__(self, src, dst): + if not self.allowed: + raise OSError("device or resource busy") + _REAL_REPLACE(src, dst) + + class TestGetCliTokenFilePath: def test_points_at_the_home_config_file(self, isolated_home): assert get_cli_token_file_path() == str(isolated_home / ".litellm" / "token.json") @@ -459,6 +480,43 @@ class TestScrubFailure: assert json.loads(path.read_text())["key"] == "sk-legacy" assert list(path.parent.glob(".tmp-*")) == [] + def test_a_rewrite_that_fails_after_the_keychain_took_the_secret_hands_it_back( + self, isolated_home, secret_vault_factory, monkeypatch + ): + """Staging can succeed and the rewrite still fail afterwards, which is the one window where + both stores hold the credential. The keychain copy goes back, so the file is left exactly as + it was found and the move can be tried again.""" + path = _write_legacy_file(isolated_home) + vault = secret_vault_factory() + monkeypatch.setattr("litellm.litellm_core_utils.private_json.os.replace", _refuse_replace) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert vault.blob is None + assert json.loads(path.read_text())["key"] == "sk-legacy" + assert list(path.parent.glob(".tmp-*")) == [] + + def test_a_rollback_the_keychain_refuses_is_finished_by_the_next_read( + self, isolated_home, secret_vault_factory, monkeypatch + ): + """A keychain that will not give back what it just took leaves the credential in both stores. + Nothing is lost by that, and nothing is abandoned either: the next read carries the move the + rest of the way, so the duplicate outlives only the condition that caused it.""" + path = _write_legacy_file(isolated_home) + vault = secret_vault_factory(erasable=False) + replace = _ReplaceThatStartsRefusing() + monkeypatch.setattr("litellm.litellm_core_utils.private_json.os.replace", replace) + + assert load_cli_token(vault=vault).key == "sk-legacy" + assert vault.blob is not None + assert json.loads(path.read_text())["key"] == "sk-legacy" + + replace.allowed = True + + assert load_cli_token(vault=vault).key == "sk-legacy" + assert json.loads(path.read_text()).get("key") is None + class TestClearCliToken: def test_removes_the_credential_from_both_stores(self, isolated_home, secret_vault_factory): @@ -599,6 +657,39 @@ class TestClearCliToken: assert vault.blob is not None assert "sk-in-file" not in _token_file(isolated_home).read_text() + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") + def test_a_file_that_can_be_neither_scrubbed_nor_removed_is_reported_not_raised( + self, isolated_home, secret_vault_factory + ): + """A `~/.litellm` gone read-only, or one left root-owned by a `sudo lite login`, refuses the + scrubbed rewrite and the removal alike. The credential is still readable on disk, which is + the one thing logging out is for, so it has to come back as an answer rather than as a + traceback the user has to read the code to understand.""" + path = _write_legacy_file(isolated_home) + path.parent.chmod(0o500) + try: + outcome = clear_cli_token(vault=secret_vault_factory()) + finally: + path.parent.chmod(0o700) + + assert isinstance(outcome, CredentialNotCleared) + assert json.loads(path.read_text())["key"] == "sk-legacy" + + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") + def test_a_metadata_file_that_will_not_go_is_not_worth_alarming_the_user_over( + self, isolated_home, secret_vault_factory + ): + """The secret was in the keychain and the keychain gave it up. What is stuck on disk names a + credential that no longer exists, so the logout it describes really did happen.""" + path = _write_metadata_only_file(isolated_home) + path.parent.chmod(0o500) + try: + outcome = clear_cli_token(vault=secret_vault_factory(blob=_blob())) + finally: + path.parent.chmod(0o700) + + assert outcome == SecretErased() + def test_is_safe_when_nothing_was_ever_stored(self, isolated_home, secret_vault_factory): assert clear_cli_token(vault=secret_vault_factory()) == SecretErased() diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 9e4ede88337..71fc1cd3e6a 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -492,6 +492,23 @@ class TestLogoutCommand: assert "still in the OS keychain" in result.output assert "Unlock your keychain" in result.output + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") + def test_logout_reports_a_token_file_it_cannot_remove(self, isolated_home, secret_vault_factory): + """`lite logout` on a read-only ~/.litellm used to end in a PermissionError traceback with + the credential still sitting in the file. The user has to be told what is left and where.""" + _write_token_file(isolated_home, key="sk-in-file") + config_dir = isolated_home / ".litellm" + config_dir.chmod(0o500) + try: + result = self.runner.invoke(logout, obj={"secret_vault": secret_vault_factory()}) + finally: + config_dir.chmod(0o700) + + assert result.exit_code == 0 + assert "Logged out successfully" not in result.output + assert "still in" in result.output + assert str(config_dir / "token.json") in result.output + def test_logout_without_the_keyring_package_still_warns_about_a_file_held_secret( self, isolated_home, secret_vault_factory ): From 4ca1f3148a9df608eda0a4b562badc2c90d25f28 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:58:53 -0700 Subject: [PATCH 16/53] fix(cli): finish a refused token file rewrite in place Taking the secret out of ~/.litellm/token.json stages a replacement and moves it into place, which needs room for a second file and a directory that will accept a new entry. A full disk refuses the first and a read-only ~/.litellm the second, and logout gave up there: it removed the file when it could, dropping the record that the keychain had never been confirmed clear, so the logout after it reported a clean keychain it never checked Shortening the file already in place needs neither, so the logout scrub and the legacy migration now fall back to overwriting it where it lies. On a read-only ~/.litellm the logout the user asked for now happens, instead of coming back with instructions to delete the file by hand --- litellm/litellm_core_utils/cli_token_utils.py | 45 ++++++++++---- litellm/litellm_core_utils/private_json.py | 15 +++++ .../test_cli_token_utils.py | 58 ++++++++++++++----- .../litellm_core_utils/test_private_json.py | 37 ++++++++++++ .../proxy/client/cli/test_auth_commands.py | 38 +++++++++--- 5 files changed, 162 insertions(+), 31 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/test_private_json.py diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index f2c0f8ed001..2f45742ce03 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -38,6 +38,7 @@ from litellm.litellm_core_utils.private_json import ( commit_staged_json, discard_staged_json, ensure_private_dir, + overwrite_private_json, stage_private_json, write_private_json, ) @@ -180,7 +181,7 @@ def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretClear: if not settled and _keep_the_unchecked_keychain_on_record(outcome, record): return outcome removal: Final = _remove_token_file() - if removal is not None and record is not None and record.key is not None: + if removal is not None and record is not None and not _scrub_file_secret(record): return removal return SecretErased() if settled else outcome @@ -198,8 +199,9 @@ def _keep_the_unchecked_keychain_on_record(outcome: SecretErase, record: CliToke Only a keychain that could not be reached leaves the question open. One that answered for itself is remembered without any help from the file, and a file it can still pair a live entry with - would leave the machine signed in to the login that was just ended. A copy that cannot be - replaced with a secret-free one is not kept either, because the secret goes first. + would leave the machine signed in to the login that was just ended. A copy that will give up + its secret neither to a staged replacement nor to an overwrite is not kept either, because the + secret goes first. """ if record is None or isinstance(outcome, SecretStranded): return False @@ -210,11 +212,12 @@ def _nothing_left_behind(outcome: SecretErase, record: CliTokenRecord | None) -> """Whether the keychain can be trusted to hold no credential of ours once the file is gone. A machine with no token file has no stored login to end, and `clear_cli_token` keeps one behind - whenever the keychain is left unconfirmed, so a missing file is real evidence rather than the - absence of it. Past that, a keychain that could not be reached is never trusted, whatever the - file looks like. Even a file holding its own secret says only that the login which wrote it had - no keychain to write to, and the login before it may well have had one: the entry that login - left outlives both the uninstalled package and the file that replaced it. `SecretStranded` is + whenever the keychain is left unconfirmed, taking the secret out in place when it cannot stage a + replacement, so a missing file is real evidence rather than the absence of it. Past that, a + keychain that could not be reached is never trusted, whatever the file looks like. Even a file + holding its own secret says only that the login which wrote it had no keychain to write to, and + the login before it may well have had one: the entry that login left outlives both the + uninstalled package and the file that replaced it. `SecretStranded` is the keychain answering for itself and outranks the file. """ match outcome: @@ -323,6 +326,11 @@ def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliToken before the keychain is handed anything. Copying the credential into a second store and only then discovering the first one cannot be cleaned would widen exposure instead of narrowing it, which is the opposite of what moving it into the keychain is for. + + A staged file that will not go into place is overwritten where it lies before the keychain is + asked to take the new entry back, so the migration finishes on a directory that would only ever + have refused it. Rolling back is the last resort, and a rollback the keychain also refuses + leaves the secret in both stores until the next read, which retries this same migration. """ if record.key is None: return None @@ -332,7 +340,7 @@ def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliToken if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): discard_staged_json(staged) return record - if not _commit_token_file(staged): + if not _commit_token_file(staged) and not _overwrite_file_secret(record): vault.erase() return record @@ -342,7 +350,24 @@ def _scrub_file_secret(record: CliTokenRecord) -> bool: if record.key is None and not record.jwt_token: return True staged: Final = _stage_scrubbed_file(record) - return staged is not None and _commit_token_file(staged) + if staged is not None and _commit_token_file(staged): + return True + return _overwrite_file_secret(record) + + +def _overwrite_file_secret(record: CliTokenRecord) -> bool: + """Take the secret out of the token file where it lies, when no replacement can be put in place. + + The atomic rewrite wants room for a second file and a directory that will accept it. A full disk + refuses the first and a read-only `~/.litellm` the second, and neither stands in the way of + shortening the file that is already there. It is worth the loss of atomicity because a partial + write reads as no login at all, which is where the refused rewrite left the next run anyway. + """ + try: + overwrite_private_json(get_cli_token_file_path(), _without_secret(record).model_dump(exclude_none=True)) + except OSError: + return False + return True def _stage_scrubbed_file(record: CliTokenRecord) -> str | None: diff --git a/litellm/litellm_core_utils/private_json.py b/litellm/litellm_core_utils/private_json.py index fbeb74aab5a..30f64c8fc27 100644 --- a/litellm/litellm_core_utils/private_json.py +++ b/litellm/litellm_core_utils/private_json.py @@ -45,6 +45,21 @@ def commit_staged_json(staged: str, path: str) -> None: raise +def overwrite_private_json(path: str, data: Mapping[str, object]) -> None: + """Rewrite a file that is already there, in place, keeping the mode it was created with. + + `write_private_json` needs room for a second file and a directory that will accept it, which is + what a full disk and a read-only `~/.litellm` respectively refuse. Shortening the file already + in place needs neither. It is not atomic, so an interrupted write leaves a partial file, and it + never creates one, so it cannot put a world-readable file where a private one was. + """ + fd: Final = os.open(path, os.O_WRONLY | os.O_TRUNC) + with os.fdopen(fd, "w") as f: + json.dump(data, f, indent=2) + f.flush() + os.fsync(f.fileno()) + + def discard_staged_json(staged: str) -> None: """Throw a staged file away when the change it was part of is abandoned""" Path(staged).unlink(missing_ok=True) diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 2830fad58b4..e3d14c6da46 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -480,12 +480,13 @@ class TestScrubFailure: assert json.loads(path.read_text())["key"] == "sk-legacy" assert list(path.parent.glob(".tmp-*")) == [] - def test_a_rewrite_that_fails_after_the_keychain_took_the_secret_hands_it_back( + def test_a_rewrite_the_directory_refuses_is_finished_in_place( self, isolated_home, secret_vault_factory, monkeypatch ): """Staging can succeed and the rewrite still fail afterwards, which is the one window where - both stores hold the credential. The keychain copy goes back, so the file is left exactly as - it was found and the move can be tried again.""" + both stores hold the credential. Shortening the file already there needs neither a second + file nor a cooperative directory, so the move finishes rather than handing the keychain copy + back and leaving the cleartext where it was.""" path = _write_legacy_file(isolated_home) vault = secret_vault_factory() monkeypatch.setattr("litellm.litellm_core_utils.private_json.os.replace", _refuse_replace) @@ -493,26 +494,30 @@ class TestScrubFailure: record = load_cli_token(vault=vault) assert record.key == "sk-legacy" - assert vault.blob is None - assert json.loads(path.read_text())["key"] == "sk-legacy" + assert vault.blob is not None + assert json.loads(path.read_text()).get("key") is None assert list(path.parent.glob(".tmp-*")) == [] + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores file permissions") def test_a_rollback_the_keychain_refuses_is_finished_by_the_next_read( self, isolated_home, secret_vault_factory, monkeypatch ): - """A keychain that will not give back what it just took leaves the credential in both stores. - Nothing is lost by that, and nothing is abandoned either: the next read carries the move the - rest of the way, so the duplicate outlives only the condition that caused it.""" + """A file that will take neither a replacement nor an overwrite, and a keychain that will not + give back what it just took, leave the credential in both stores. Nothing is lost by that, + and nothing is abandoned either: the next read carries the move the rest of the way, so the + duplicate outlives only the conditions that caused it.""" path = _write_legacy_file(isolated_home) vault = secret_vault_factory(erasable=False) replace = _ReplaceThatStartsRefusing() monkeypatch.setattr("litellm.litellm_core_utils.private_json.os.replace", replace) + path.chmod(0o400) assert load_cli_token(vault=vault).key == "sk-legacy" assert vault.blob is not None assert json.loads(path.read_text())["key"] == "sk-legacy" replace.allowed = True + path.chmod(0o600) assert load_cli_token(vault=vault).key == "sk-legacy" assert json.loads(path.read_text()).get("key") is None @@ -657,24 +662,49 @@ class TestClearCliToken: assert vault.blob is not None assert "sk-in-file" not in _token_file(isolated_home).read_text() - @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") - def test_a_file_that_can_be_neither_scrubbed_nor_removed_is_reported_not_raised( + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores file permissions") + def test_a_file_that_gives_up_neither_its_secret_nor_itself_is_reported_not_raised( self, isolated_home, secret_vault_factory ): - """A `~/.litellm` gone read-only, or one left root-owned by a `sudo lite login`, refuses the - scrubbed rewrite and the removal alike. The credential is still readable on disk, which is - the one thing logging out is for, so it has to come back as an answer rather than as a - traceback the user has to read the code to understand.""" + """A `~/.litellm` gone read-only refuses the staged rewrite and the removal, and a token file + left read-only with it, as a `sudo lite login` leaves both, refuses the overwrite too. The + credential is still readable on disk, which is the one thing logging out is for, so it has to + come back as an answer rather than as a traceback the user has to read the code to + understand.""" path = _write_legacy_file(isolated_home) + path.chmod(0o400) path.parent.chmod(0o500) try: outcome = clear_cli_token(vault=secret_vault_factory()) finally: path.parent.chmod(0o700) + path.chmod(0o600) assert isinstance(outcome, CredentialNotCleared) assert json.loads(path.read_text())["key"] == "sk-legacy" + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") + def test_a_directory_that_takes_no_new_file_still_gives_up_the_secret_in_the_old_one( + self, isolated_home, secret_vault_factory + ): + """A read-only `~/.litellm` accepts no replacement token file and no removal of the one it + has, and still lets that one be shortened. The secret goes, the file stays as the note that + the keychain went unchecked, and the logout after it warns again instead of reading the gap + the removal would have left as a clean keychain. + + The key is a realistic length so the file genuinely shrinks: a rewrite in place that leaves + the tail of the old contents behind hands the next run a file it cannot parse.""" + path = _write_legacy_file(isolated_home, key="sk-" + "a" * 700) + vault = secret_vault_factory(available=False, failure=KeyringUnreachable()) + path.parent.chmod(0o500) + try: + assert clear_cli_token(vault=vault) == KeyringUnreachable() + assert clear_cli_token(vault=vault) == KeyringUnreachable() + finally: + path.parent.chmod(0o700) + + assert json.loads(path.read_text()).get("key") is None + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") def test_a_metadata_file_that_will_not_go_is_not_worth_alarming_the_user_over( self, isolated_home, secret_vault_factory diff --git a/tests/test_litellm/litellm_core_utils/test_private_json.py b/tests/test_litellm/litellm_core_utils/test_private_json.py new file mode 100644 index 00000000000..cedff61959f --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_private_json.py @@ -0,0 +1,37 @@ +import json +import os +import stat + +import pytest + +from litellm.litellm_core_utils.private_json import overwrite_private_json, write_private_json + + +class TestOverwritePrivateJson: + def test_replaces_the_contents_of_the_file_already_there(self, tmp_path): + path = tmp_path / "token.json" + write_private_json(str(path), {"key": "sk-" + "a" * 700}) + + overwrite_private_json(str(path), {"user_id": "u-1"}) + + assert json.loads(path.read_text()) == {"user_id": "u-1"} + + def test_refuses_to_create_the_file_it_was_asked_to_rewrite(self, tmp_path): + """This is the one writer that does not go through a private temp file, so a path it creates + would land with whatever the umask allows. Refusing keeps it unable to put a world-readable + file where the caller believed a private one already was.""" + path = tmp_path / "token.json" + + with pytest.raises(FileNotFoundError): + overwrite_private_json(str(path), {"user_id": "u-1"}) + + assert not path.exists() + + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores file permissions") + def test_keeps_the_owner_only_mode_the_file_was_created_with(self, tmp_path): + path = tmp_path / "token.json" + write_private_json(str(path), {"key": "sk-live"}) + + overwrite_private_json(str(path), {"user_id": "u-1"}) + + assert stat.S_IMODE(path.stat().st_mode) == 0o600 diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 71fc1cd3e6a..8191a8edca7 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -492,12 +492,37 @@ class TestLogoutCommand: assert "still in the OS keychain" in result.output assert "Unlock your keychain" in result.output - @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") - def test_logout_reports_a_token_file_it_cannot_remove(self, isolated_home, secret_vault_factory): - """`lite logout` on a read-only ~/.litellm used to end in a PermissionError traceback with - the credential still sitting in the file. The user has to be told what is left and where.""" + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores file permissions") + def test_logout_reports_a_token_file_it_cannot_clear(self, isolated_home, secret_vault_factory): + """`lite logout` on a read-only ~/.litellm holding a read-only token file used to end in a + PermissionError traceback with the credential still sitting in the file. The user has to be + told what is left and where.""" _write_token_file(isolated_home, key="sk-in-file") config_dir = isolated_home / ".litellm" + path = config_dir / "token.json" + path.chmod(0o400) + config_dir.chmod(0o500) + try: + result = self.runner.invoke(logout, obj={"secret_vault": secret_vault_factory()}) + finally: + config_dir.chmod(0o700) + path.chmod(0o600) + + assert result.exit_code == 0 + assert "Logged out successfully" not in result.output + assert "still in" in result.output + assert str(config_dir / "token.json") in result.output + + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") + def test_logout_on_a_read_only_directory_still_takes_the_secret_out_of_the_file( + self, isolated_home, secret_vault_factory + ): + """A ~/.litellm that will accept no replacement file and no removal still lets the file it + has be shortened, so the logout the user asked for happens rather than being handed back to + them with instructions.""" + _write_token_file(isolated_home, key="sk-in-file") + config_dir = isolated_home / ".litellm" + path = config_dir / "token.json" config_dir.chmod(0o500) try: result = self.runner.invoke(logout, obj={"secret_vault": secret_vault_factory()}) @@ -505,9 +530,8 @@ class TestLogoutCommand: config_dir.chmod(0o700) assert result.exit_code == 0 - assert "Logged out successfully" not in result.output - assert "still in" in result.output - assert str(config_dir / "token.json") in result.output + assert "Logged out successfully" in result.output + assert "sk-in-file" not in path.read_text() def test_logout_without_the_keyring_package_still_warns_about_a_file_held_secret( self, isolated_home, secret_vault_factory From b142d1d76576221e394893c8e7abaf3254e167d3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:59:20 -0700 Subject: [PATCH 17/53] fix(cli): stop whoami calling an unreadable credential authenticated `lite whoami` led with "Authenticated" whenever a token file was on disk, even when the keychain holding the credential would not give it up. The notice about that sat below the account lines, so the session read as a working one and sent the user looking for the problem anywhere but the keychain --- litellm/proxy/client/cli/commands/auth.py | 2 +- .../proxy/client/cli/test_auth_commands.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index b3a7db4bed4..ac330ad6892 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -867,7 +867,7 @@ def whoami(ctx: click.Context): click.echo("Not authenticated. Run 'lite login' to authenticate.") return - click.echo("Authenticated") + click.echo("Authenticated" if token_data.key is not None else "Signed in, but the credential cannot be read") click.echo(f"User Email: {token_data.user_email or 'Unknown'}") click.echo(f"User ID: {token_data.user_id or 'Unknown'}") click.echo(f"User Role: {token_data.user_role or 'Unknown'}") diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 8191a8edca7..1dd8a7ead92 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -1137,7 +1137,12 @@ class TestKeychainBackedCommands: assert "could not be read" in result.output assert "lite login" in result.output - def test_whoami_flags_a_locked_keychain(self, isolated_home, secret_vault_factory): + def test_whoami_does_not_call_a_credential_it_cannot_read_authenticated( + self, isolated_home, secret_vault_factory + ): + """A login whose secret is stuck in an unreachable keychain authenticates nothing. Leading + with "Authenticated" and a token age reads as a working session, and sends the user looking + for the problem somewhere other than the keychain the notice underneath names.""" _write_home_json( isolated_home, "token.json", @@ -1147,7 +1152,8 @@ class TestKeychainBackedCommands: result = self.runner.invoke(whoami, obj=obj) - assert "Authenticated" in result.output + assert "Authenticated" not in result.output + assert "the credential cannot be read" in result.output assert "could not be read" in result.output def test_whoami_names_the_kill_switch_rather_than_a_missing_package( From fe11202c2df1a6373ed3230e22847a76e157cb0d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:28:28 -0700 Subject: [PATCH 18/53] fix(cli): write the logout note again when the file holding it had to go When a full disk refuses the replacement file and a read-only token file refuses the rewrite in place, the only way left to get the secret off disk is to remove the file carrying it. That file was also the note saying the keychain went unchecked, so its absence made the next logout read a keychain that was never confirmed as one already known to be clean. Removing it is what frees the room the replacement was refused for, so the note is written again on the way out and the logout after this one still warns. --- litellm/litellm_core_utils/cli_token_utils.py | 32 ++++++++++-- .../test_cli_token_utils.py | 52 ++++++++++++++++++- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 2f45742ce03..69a77a36882 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -173,7 +173,8 @@ def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretClear: what lets a later run tell a machine with a credential it cannot reach apart from one that never had a login at all, and taking it away would leave the next logout answering the warning this one just issued with a false all-clear. The secret goes either way, and a file that will give up - neither its copy nor itself outranks whatever the keychain had to say. + neither its copy nor itself is removed rather than kept, with the note written again afterwards + so the warning still outlives this run. """ outcome: Final = vault.erase() record: Final = _read_token_file() @@ -183,6 +184,8 @@ def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretClear: removal: Final = _remove_token_file() if removal is not None and record is not None and not _scrub_file_secret(record): return removal + if removal is None and record is not None and _the_keychain_went_unchecked(outcome): + _write_the_note_the_removal_took_with_it(record) return SecretErased() if settled else outcome @@ -194,6 +197,19 @@ def _remove_token_file() -> CredentialNotCleared | None: return None +def _write_the_note_the_removal_took_with_it(record: CliTokenRecord) -> None: + """Put the secret-free note back after the file carrying it had to go to get the secret off disk. + + Reaching here means neither rewrite would take, so the file went instead, and its absence is + what the next logout would read as a keychain already known to be clean. Removing it is also + what frees the room the rewrite was refused for, so the note usually lands on this second try. + When it does not, the warning this logout printed is the only one the user gets. + """ + staged: Final = _stage_scrubbed_file(record) + if staged is not None: + _commit_token_file(staged) + + def _keep_the_unchecked_keychain_on_record(outcome: SecretErase, record: CliTokenRecord | None) -> bool: """Whether the token file, stripped of its secret, is worth keeping as the note that says so. @@ -203,17 +219,27 @@ def _keep_the_unchecked_keychain_on_record(outcome: SecretErase, record: CliToke its secret neither to a staged replacement nor to an overwrite is not kept either, because the secret goes first. """ - if record is None or isinstance(outcome, SecretStranded): + if record is None or not _the_keychain_went_unchecked(outcome): return False return _scrub_file_secret(record) +def _the_keychain_went_unchecked(outcome: SecretErase) -> bool: + """Whether the keychain neither confirmed the erase nor answered that it still holds the secret""" + match outcome: + case SecretErased() | SecretStranded(): + return False + case KeyringDisabled() | KeyringNotInstalled() | KeyringUnreachable(): + return True + + def _nothing_left_behind(outcome: SecretErase, record: CliTokenRecord | None) -> bool: """Whether the keychain can be trusted to hold no credential of ours once the file is gone. A machine with no token file has no stored login to end, and `clear_cli_token` keeps one behind whenever the keychain is left unconfirmed, taking the secret out in place when it cannot stage a - replacement, so a missing file is real evidence rather than the absence of it. Past that, a + replacement and writing the note again when the file holding it had to go, so a missing file is + real evidence rather than the absence of it. Past that, a keychain that could not be reached is never trusted, whatever the file looks like. Even a file holding its own secret says only that the login which wrote it had no keychain to write to, and the login before it may well have had one: the entry that login left outlives both the diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index e3d14c6da46..b8fcf3618cf 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -1,7 +1,9 @@ +import errno import json import os import stat import sys +import tempfile import threading import time @@ -82,6 +84,25 @@ def _blob(base_url=SERVER, key="sk-vault", jwt_token=""): return json.dumps({"base_url": base_url, "key": key, "jwt_token": jwt_token}) +_REAL_MKSTEMP = tempfile.mkstemp + + +class _MkstempThatNeedsTheOldFileGone: + """A disk with exactly one token file's worth of room left on it. + + Staging a replacement needs room for a second file, which is what a full disk refuses. Removing + the file already there is what gives that room back. + """ + + def __init__(self, path): + self.path = path + + def __call__(self, *args, **kwargs): + if self.path.exists(): + raise OSError(errno.ENOSPC, "No space left on device") + return _REAL_MKSTEMP(*args, **kwargs) + + _REAL_REPLACE = os.replace @@ -553,7 +574,7 @@ class TestClearCliToken: assert load_cli_token(vault=vault) is None @pytest.mark.parametrize( - "failure", [KeyringDisabled(), KeyringUnreachable(), KeyringDiscardsWrites()] + "failure", [KeyringDisabled(), KeyringUnreachable(), KeyringNotInstalled()] ) def test_a_secret_in_the_file_is_no_evidence_about_a_keychain_that_exists( self, isolated_home, secret_vault_factory, failure @@ -561,7 +582,10 @@ class TestClearCliToken: """Store a secret in the keychain, sign in again while the keychain is unusable so the new secret lands in the file, then log out while it is still unusable. The file now carries its own secret and the first login's entry is still there, so reading the file as proof of a - clean keychain reports a logout that did not happen.""" + clean keychain reports a logout that did not happen. + + The three unusable states are the whole of what an erase can answer besides erased and + stranded; a backend that keeps nothing it is given is something only a write finds out.""" _write_legacy_file(isolated_home) vault = secret_vault_factory(available=False, failure=failure) @@ -705,6 +729,30 @@ class TestClearCliToken: assert json.loads(path.read_text()).get("key") is None + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores file permissions") + def test_a_note_the_logout_had_to_remove_is_written_again_for_the_next_one( + self, isolated_home, secret_vault_factory, monkeypatch + ): + """A full disk refuses the replacement file and a read-only token file refuses the rewrite + in place, so the only way left to get the secret off disk is to remove the file carrying it. + That file was also the note saying the keychain went unchecked, and its absence is what the + next logout would read as a keychain already known to be clean. + + Removing it is what frees the room the replacement was refused for, so the note is written + again on the way out and the logout after this one still warns.""" + path = _write_legacy_file(isolated_home) + path.chmod(0o400) + monkeypatch.setattr( + "litellm.litellm_core_utils.private_json.tempfile.mkstemp", + _MkstempThatNeedsTheOldFileGone(path), + ) + vault = secret_vault_factory(available=False, failure=KeyringUnreachable()) + + assert clear_cli_token(vault=vault) == KeyringUnreachable() + assert clear_cli_token(vault=vault) == KeyringUnreachable() + + assert json.loads(path.read_text()).get("key") is None + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") def test_a_metadata_file_that_will_not_go_is_not_worth_alarming_the_user_over( self, isolated_home, secret_vault_factory From a2928efc75032f9a20a7685fce1ae3ffd8dc4c9d Mon Sep 17 00:00:00 2001 From: Mateo Date: Thu, 20 Aug 2026 03:45:00 -0700 Subject: [PATCH 19/53] test(cli): cover the keyless token record and keep keyring to the cli extra `lite up` treats a token record whose key the keychain would not hand over as no login at all, and that clause had no test: every existing freshness test passed a record carrying a real key, so deleting the clause left the whole suite green The base install smoke check now also asserts keyring is absent, which is what makes the lazy import in cli_keyring meaningful. keyring ships in the cli extra only, so a plain `pip install litellm` must not be able to reach it --- .../base_sdk_tests/check_base_sdk_install.py | 2 +- .../proxy/client/cli/test_up_commands.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/base_sdk_tests/check_base_sdk_install.py b/tests/base_sdk_tests/check_base_sdk_install.py index 723f30cad76..6b38de75e2e 100644 --- a/tests/base_sdk_tests/check_base_sdk_install.py +++ b/tests/base_sdk_tests/check_base_sdk_install.py @@ -11,7 +11,7 @@ import sys import traceback from collections.abc import Callable -EXTRAS_ONLY_MODULES = ("fastapi", "uvicorn") +EXTRAS_ONLY_MODULES = ("fastapi", "uvicorn", "keyring") def _require(condition: bool, message: str) -> None: diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index aebf441f777..a8d81f1c4bb 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -262,6 +262,28 @@ class TestEnsureFreshLogin: assert login_calls == ["http://proxy-b:4000"] + def test_forces_a_fresh_login_when_the_cached_token_has_no_readable_key(self, monkeypatch): + monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: True) + tokens = iter( + [ + _token(None, "http://proxy-a:4000"), + _token("sk-a", "http://proxy-a:4000"), + ] + ) + monkeypatch.setattr(up_module, "load_cli_token", lambda **_: next(tokens)) + monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) + login_calls = [] + + @click.pass_context + def fake_login(ctx): + login_calls.append(ctx.obj["base_url"]) + + monkeypatch.setattr(up_module, "login", fake_login) + + _ensure_fresh_login(_make_ctx("http://proxy-a:4000")) + + assert login_calls == ["http://proxy-a:4000"] + def test_fails_cleanly_non_interactively_when_only_a_different_proxys_token_is_cached(self, monkeypatch): monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: False) monkeypatch.setattr(up_module, "load_cli_token", lambda **_: _token("sk-a", "http://proxy-a:4000")) From 5099379e034533c16793d77ad9548f68b3ed07f1 Mon Sep 17 00:00:00 2001 From: Mateo Date: Thu, 20 Aug 2026 04:12:23 -0700 Subject: [PATCH 20/53] fix(cli): stop asking a keychain that already stopped answering A pre-flight that times out leaves its write parked inside the keychain, holding it against every later call, so the next read blocks on the main thread with no timeout of its own. Anything that resolves the credential more than once in a process hits it: an SDK Client built a second time never returns. The vault now remembers the silence and reports the keychain unreachable for the rest of the process rather than queueing behind the parked call. --- litellm/litellm_core_utils/cli_keyring.py | 16 ++++++- .../test_cli_token_utils.py | 47 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index 8da3e5226d4..70b1773739d 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -18,7 +18,7 @@ throwaway value, because a keychain can answer neither way and block forever. import os import threading from contextlib import suppress -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Final, Protocol, TypeAlias KEYRING_SERVICE: Final = "litellm-cli" @@ -152,11 +152,20 @@ def _forget_the_preflight(api: KeyringApi) -> None: @dataclass(frozen=True, slots=True) class KeyringVault: - """The OS keychain, reached through the optional `keyring` package.""" + """The OS keychain, reached through the optional `keyring` package. + + A keychain that let the pre-flight time out is not asked anything else for the rest of the + process. The probe that timed out is still sitting in the keychain on a thread of its own, and + it holds the keychain against every later call, so the read after it would block on the main + thread with no timeout to save it. One silence is answer enough. + """ preflight_timeout_seconds: float = _PREFLIGHT_TIMEOUT_SECONDS + stopped_answering: threading.Event = field(default_factory=threading.Event, compare=False, repr=False) def read(self) -> SecretRead: + if self.stopped_answering.is_set(): + return KeyringUnreachable() api: Final = _keyring_api() if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): return api @@ -177,10 +186,13 @@ class KeyringVault: The keychain is pre-flighted first, because one that blocks rather than answering would otherwise hang `lite login` outright. """ + if self.stopped_answering.is_set(): + return KeyringUnreachable() api: Final = _keyring_api() if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): return api if not _answers_a_write(api, self.preflight_timeout_seconds): + self.stopped_answering.set() return KeyringUnreachable() _forget_the_preflight(api) try: diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index b8fcf3618cf..c6fee6d0c25 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -832,6 +832,26 @@ class _NeverAnsweringKeyringModule(_FakeKeyringModule): threading.Event().wait() +class _KeychainHeldByABlockedWrite(_NeverAnsweringKeyringModule): + """The same keychain, plus what the blocked write does to everything after it: the stuck call + holds the keychain, so every later read blocks behind it too.""" + + def get_password(self, service_name, username): + self.calls.append(("get", service_name, username)) + if self.blocked.is_set(): + threading.Event().wait() + return self.stored + + +def _answered_within(seconds, call): + answers = [] + worker = threading.Thread(target=lambda: answers.append(call()), daemon=True) + worker.start() + worker.join(seconds) + assert not worker.is_alive(), f"{call.__qualname__} never returned" + return answers[0] + + @pytest.fixture def install_fake_keyring(monkeypatch): def _install(fake): @@ -927,6 +947,33 @@ class TestKeyringVault: assert [call[2] for call in fake.calls] == [KEYRING_PREFLIGHT_ACCOUNT] + def test_a_keychain_that_stopped_answering_is_not_asked_again(self, install_fake_keyring): + """The write that timed out is still holding the keychain when we give up on it, so the + call after it is the one that hangs, and read has nothing to time out against. Anything + resolving the credential more than once in a process hits that: an SDK client built twice + pays the pre-flight timeout on the first build and never returns from the second.""" + install_fake_keyring(_KeychainHeldByABlockedWrite()) + vault = KeyringVault(preflight_timeout_seconds=0.05) + + assert vault.write("blob-1") == KeyringUnreachable() + + assert _answered_within(5, vault.read) == KeyringUnreachable() + assert _answered_within(5, vault.erase) == KeyringUnreachable() + assert _answered_within(5, lambda: vault.write("blob-2")) == KeyringUnreachable() + + def test_a_keychain_that_stopped_answering_leaves_the_credential_in_the_file( + self, isolated_home, install_fake_keyring + ): + """The end of the same story: giving up on the keychain has to leave a login that still + works, and loading it back must not go asking the keychain that already stopped answering.""" + install_fake_keyring(_KeychainHeldByABlockedWrite()) + vault = KeyringVault(preflight_timeout_seconds=0.05) + + outcome = save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-only-copy"), vault=vault) + + assert outcome == KeyringUnreachable() + assert _answered_within(5, lambda: load_cli_token(vault=vault)).key == "sk-only-copy" + def test_a_login_survives_a_keychain_that_never_answers(self, isolated_home, install_fake_keyring): """The end of the same story: the credential still has to be usable afterwards.""" install_fake_keyring(_NeverAnsweringKeyringModule()) From 71a583390a4efe0dd171a249b62f895d73a4ad39 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 04:29:34 -0700 Subject: [PATCH 21/53] fix(cli): pick the credential by the sign-in it came from A login the keychain accepted whose token file could not be replaced left the superseded secret on disk, and the next load preferred the file unconditionally, so it served the old credential and erased the new one from the keychain on the way past. The keychain entry now carries the timestamp of the sign-in that minted it, and the two stores are compared on that instead. --- litellm/litellm_core_utils/cli_token_utils.py | 52 ++++++++++++------- litellm/proxy/client/cli/commands/auth.py | 5 +- .../test_cli_token_utils.py | 35 ++++++++++++- 3 files changed, 69 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 69a77a36882..134e7e2331c 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -105,7 +105,8 @@ class CliTokenSecret(BaseModel): `base_url` is duplicated from the metadata file purely as a pairing tag: a secret minted for one server is never handed to another, even if the - metadata file is edited underneath us. + metadata file is edited underneath us. `timestamp` is the sign-in this + secret came from, which is what decides it against a secret still on disk. """ model_config = ConfigDict(frozen=True) @@ -113,6 +114,7 @@ class CliTokenSecret(BaseModel): base_url: str key: str jwt_token: str = "" + timestamp: float = 0.0 def get_cli_token_file_path() -> str: @@ -145,11 +147,7 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN staged: Final = _stage_token_file(_without_secret(record)) if isinstance(staged, CredentialNotSaved): return staged - outcome: Final = ( - SecretStored() - if record.key is None - else vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)) - ) + outcome: Final = SecretStored() if record.key is None else vault.write(_encode_secret(record, record.key)) if isinstance(outcome, SecretStored): return outcome if _commit_token_file(staged) else CredentialNotRecorded() discard_staged_json(staged) @@ -330,19 +328,24 @@ def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecor def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) -> CliTokenRecord | None: """Resolve the credential when both stores hold one. - A secret still on disk is the fresher of the two, because it is only left there when the - keychain write that should have removed it failed, so it outranks the vault entry. + The sign-in each secret came from decides it, because either store can be the stale one. A + secret is usually left on disk by a keychain that would not take it, which makes the file the + fresher of the two. It is the older one when a login the keychain did take could not replace + the file afterwards, and serving that one would put a superseded credential back in use. """ - if record.key is not None: - return _migrate_file_secret(record, vault) - try: - secret: Final = CliTokenSecret.model_validate_json(blob) - except ValidationError: - return _migrate_file_secret(record, vault) - if secret.base_url != record.base_url: + secret: Final = _decode_secret(blob, record.base_url) + if secret is None or (record.key is not None and secret.timestamp <= record.timestamp): return _migrate_file_secret(record, vault) _scrub_file_secret(record) - return record.model_copy(update=MappingProxyType({"key": secret.key, "jwt_token": secret.jwt_token})) + return record.model_copy( + update=MappingProxyType( + { + "key": secret.key, + "jwt_token": secret.jwt_token, + "timestamp": max(secret.timestamp, record.timestamp), + } + ) + ) def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: @@ -363,7 +366,7 @@ def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliToken staged: Final = _stage_scrubbed_file(record) if staged is None: return record - if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): + if not isinstance(vault.write(_encode_secret(record, record.key)), SecretStored): discard_staged_json(staged) return record if not _commit_token_file(staged) and not _overwrite_file_secret(record): @@ -422,8 +425,19 @@ def _without_secret(record: CliTokenRecord) -> CliTokenRecord: return record.model_copy(update=MappingProxyType({"key": None, "jwt_token": ""})) -def _encode_secret(base_url: str, key: str, jwt_token: str) -> str: - return CliTokenSecret(base_url=base_url, key=key, jwt_token=jwt_token).model_dump_json() +def _encode_secret(record: CliTokenRecord, key: str) -> str: + return CliTokenSecret( + base_url=record.base_url, key=key, jwt_token=record.jwt_token, timestamp=record.timestamp + ).model_dump_json() + + +def _decode_secret(blob: str, base_url: str) -> CliTokenSecret | None: + """The keychain entry, when it is one this metadata file may be paired with""" + try: + secret: Final = CliTokenSecret.model_validate_json(blob) + except ValidationError: + return None + return secret if secret.base_url == base_url else None def _write_token_file(record: CliTokenRecord) -> None: diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index ac330ad6892..e5d90b9b640 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -131,8 +131,9 @@ def storage_notice(outcome: SecretSave) -> str: case CredentialNotRecorded(): return ( f"Signed in, and the credential is in your OS keychain, but {path} could not be " - "replaced, so this machine may still be using your previous login. Run 'lite login' " - "again once that path is writable, or 'lite logout' to clear both." + "replaced, so it still describes your previous login and may still hold its " + "credential. Run 'lite login' again once that path is writable, or 'lite logout' " + "to clear both." ) diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index c6fee6d0c25..4fc835d2db1 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -80,8 +80,8 @@ def _write_metadata_only_file(home): return path -def _blob(base_url=SERVER, key="sk-vault", jwt_token=""): - return json.dumps({"base_url": base_url, "key": key, "jwt_token": jwt_token}) +def _blob(base_url=SERVER, key="sk-vault", jwt_token="", timestamp=0.0): + return json.dumps({"base_url": base_url, "key": key, "jwt_token": jwt_token, "timestamp": timestamp}) _REAL_MKSTEMP = tempfile.mkstemp @@ -210,6 +210,37 @@ class TestLoadCliToken: assert json.loads(vault.blob)["key"] == "sk-fresh" assert "key" not in json.loads(path.read_text()) + def test_a_login_the_file_could_not_record_is_the_one_that_gets_used( + self, isolated_home, secret_vault_factory + ): + """A login the keychain took and the file could not be pointed at afterwards leaves the + superseded secret sitting on disk in front of the fresh one. Serving the file's copy would + put a credential the user just replaced, and may well have just revoked, back into every + request, and would overwrite the keychain with it on the way past.""" + path = _write_legacy_file(isolated_home, key="sk-superseded", timestamp=1000.0) + vault = secret_vault_factory(blob=_blob(key="sk-fresh", timestamp=2000.0)) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-fresh" + assert record.timestamp == 2000.0 + assert json.loads(vault.blob)["key"] == "sk-fresh" + assert "key" not in json.loads(path.read_text()) + + def test_a_secret_written_to_disk_after_the_keychain_entry_still_wins( + self, isolated_home, secret_vault_factory + ): + """The other direction of the same rule, which is the common one: a login that fell back to + the file because the keychain refused it is newer than whatever the keychain kept.""" + path = _write_legacy_file(isolated_home, key="sk-fresh", timestamp=2000.0) + vault = secret_vault_factory(blob=_blob(key="sk-stale", timestamp=1000.0)) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-fresh" + assert json.loads(vault.blob)["key"] == "sk-fresh" + assert "key" not in json.loads(path.read_text()) + def test_a_disk_secret_survives_when_the_stale_vault_refuses_the_rewrite( self, isolated_home, secret_vault_factory ): From b9ce630b0e47480676b943ba2208133509afac3b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 04:36:32 -0700 Subject: [PATCH 22/53] docs(cli): say why a refused scrub does not roll the keychain back The two stores hold different credentials on that path, so the rollback a migration does would hand the superseded one back out. The login that could not replace the file already named the state, and logout reports it too. --- litellm/litellm_core_utils/cli_token_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 134e7e2331c..085448f8b63 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -332,6 +332,12 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) - secret is usually left on disk by a keychain that would not take it, which makes the file the fresher of the two. It is the older one when a login the keychain did take could not replace the file afterwards, and serving that one would put a superseded credential back in use. + + A scrub the file refuses leaves that superseded secret where it lies, which is the state the + login already named when it could not replace the file, and which `lite logout` reports rather + than counting as a clean sweep. Rolling the vault back the way a migration does is not the + answer here, because the two stores hold different credentials and the rollback would hand the + superseded one back out. """ secret: Final = _decode_secret(blob, record.base_url) if secret is None or (record.key is not None and secret.timestamp <= record.timestamp): From a329dfbb45ea3ec7ea39bb6621d0093f4862094b Mon Sep 17 00:00:00 2001 From: Mateo Edgeton Date: Thu, 20 Aug 2026 04:53:09 -0700 Subject: [PATCH 23/53] fix(cli): keep each sign-in stamped past the one it replaces The stamp in the keychain entry is what decides that secret against one still sitting in the token file, and it came straight off the wall clock. A clock that stepped backwards between two logins therefore handed the win to the older of them: a login the keychain took but the token file could not be pointed at was resolved back to the credential it replaced, and the fresh one was erased from the keychain on the way past. save_cli_token now reads the stamp already on disk and pins the new sign-in just above it, so the ordering never depends on the clock having moved forwards. On a clock that did, this changes nothing. --- litellm/litellm_core_utils/cli_token_utils.py | 26 ++++++++++-- .../test_cli_token_utils.py | 40 +++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 085448f8b63..da8b80ab598 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -12,6 +12,7 @@ first time it reads one. This module has no dependencies on proxy code and can be safely imported at the SDK level. """ +import math import time from dataclasses import dataclass from pathlib import Path @@ -144,14 +145,29 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN keychain has already taken the new secret, and it reports itself as such rather than claiming the previous login survived. """ - staged: Final = _stage_token_file(_without_secret(record)) + stamped: Final = _stamped_past_the_login_on_disk(record) + staged: Final = _stage_token_file(_without_secret(stamped)) if isinstance(staged, CredentialNotSaved): return staged - outcome: Final = SecretStored() if record.key is None else vault.write(_encode_secret(record, record.key)) + outcome: Final = SecretStored() if stamped.key is None else vault.write(_encode_secret(stamped, stamped.key)) if isinstance(outcome, SecretStored): return outcome if _commit_token_file(staged) else CredentialNotRecorded() discard_staged_json(staged) - return _keep_the_secret_in_the_file(record, outcome) + return _keep_the_secret_in_the_file(stamped, outcome) + + +def _stamped_past_the_login_on_disk(record: CliTokenRecord) -> CliTokenRecord: + """Keep a sign-in's stamp ahead of the one it replaces, whatever the clock did in between. + + The stamp is what decides a keychain secret against one still on disk, so a clock that stepped + backwards between two logins would hand the older of them the win and put a superseded + credential back in use. The file already names the login being replaced, and pinning the new + stamp just past it costs one read that changes nothing on a clock that only moves forwards. + """ + previous: Final = _read_token_file() + if previous is None or previous.timestamp < record.timestamp: + return record + return record.model_copy(update=MappingProxyType({"timestamp": math.nextafter(previous.timestamp, math.inf)})) def _keep_the_secret_in_the_file(record: CliTokenRecord, outcome: SecretWrite) -> SecretSave: @@ -331,7 +347,9 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) - The sign-in each secret came from decides it, because either store can be the stale one. A secret is usually left on disk by a keychain that would not take it, which makes the file the fresher of the two. It is the older one when a login the keychain did take could not replace - the file afterwards, and serving that one would put a superseded credential back in use. + the file afterwards, and serving that one would put a superseded credential back in use. Equal + stamps are one login sitting in both stores, left by a migration whose scrub was refused, so + that branch retries the migration rather than trading one credential for another. A scrub the file refuses leaves that superseded secret where it lies, which is the state the login already named when it could not replace the file, and which `lite logout` reports rather diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 4fc835d2db1..d9fc17f908f 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -488,6 +488,46 @@ class TestSaveCliToken: assert path.read_text() == before assert list(path.parent.glob(".tmp-*")) == [] + def test_a_login_is_stamped_past_the_one_it_replaces_even_on_a_clock_that_went_back( + self, isolated_home, secret_vault_factory + ): + """The stamp is what decides the keychain secret against the one on disk, so a login that + carries an earlier wall clock than the login before it must not be filed as the older of + the two.""" + _write_legacy_file(isolated_home, key="sk-old", timestamp=2000.0) + vault = secret_vault_factory() + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new", timestamp=1000.0), vault=vault) + + assert json.loads(vault.blob)["timestamp"] > 2000.0 + + def test_a_clock_that_went_back_does_not_hand_the_win_to_the_superseded_login( + self, isolated_home, secret_vault_factory + ): + """The disk state a login reports as CredentialNotRecorded: the keychain took the new + secret and the file still holds the previous one. Reading it back has to produce the login + that was just made, and an earlier wall clock is no reason to serve the one it replaced.""" + _write_legacy_file(isolated_home, key="sk-superseded", timestamp=2000.0) + vault = secret_vault_factory() + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-fresh", timestamp=1000.0), vault=vault) + _write_legacy_file(isolated_home, key="sk-superseded", timestamp=2000.0) + + assert load_cli_token(vault=vault).key == "sk-fresh" + + def test_a_login_on_a_clock_that_moved_forwards_keeps_its_own_time( + self, isolated_home, secret_vault_factory + ): + """Pinning the stamp above the previous login is only ever a floor. The ordinary case has + to record when the user actually signed in, because that is what decides expiry.""" + _write_legacy_file(isolated_home, key="sk-old", timestamp=1000.0) + vault = secret_vault_factory() + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new", timestamp=2000.0), vault=vault) + + assert json.loads(vault.blob)["timestamp"] == 2000.0 + assert json.loads(_token_file(isolated_home).read_text())["timestamp"] == 2000.0 + class TestScrubFailure: """A keychain that took the secret while the file kept it is the worst of both stores: the From 8a8e8dc8edcf819041a5ff23b6a30f5a4939347e Mon Sep 17 00:00:00 2001 From: Mateo Edgeton Date: Thu, 20 Aug 2026 04:53:09 -0700 Subject: [PATCH 24/53] docs(deps): say that the cli extra pulls cryptography on linux The comment above the extra named cryptography as one of the heavy imports a thin install leaves out. That stopped being true when keyring joined the extra: on Linux it reaches the Secret Service through secretstorage, which depends on cryptography. --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 64adb9cd595..09a69f3771e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,8 +78,10 @@ proxy = [ "expression>=5.6.0,<6.0", ] # Thin client install for the `lite` CLI on developer laptops. The CLI's heavy -# imports (fastapi, cryptography, ...) are all guarded, so it runs on the base -# SDK plus just these five; none of the server runtime in `proxy` is pulled in. +# imports are all guarded, so it runs on the base SDK plus just these five, and +# none of the server runtime in `proxy` is pulled in. On Linux, +# keyring reaches the Secret Service through secretstorage, which brings +# cryptography with it. cli = [ "rich>=13.9.4,<14.0", "pyyaml>=6.0.3,<7.0", From fb69fcf765997610e7f32fc041200cd5b5217e3a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 05:07:21 -0700 Subject: [PATCH 25/53] fix(cli): stamp each sign-in past the keychain as well as the file A login the keychain took but the token file could not record leaves the keychain naming a later sign-in than the file does. Reading only the file then stamps the next login below that keychain entry, and a clock that went back far enough puts the superseded credential back in use. --- litellm/litellm_core_utils/cli_token_utils.py | 40 +++++++++++++++---- .../test_cli_token_utils.py | 13 ++++++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index da8b80ab598..c0e57a737e4 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -145,7 +145,7 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN keychain has already taken the new secret, and it reports itself as such rather than claiming the previous login survived. """ - stamped: Final = _stamped_past_the_login_on_disk(record) + stamped: Final = _stamped_past_every_stored_login(record, vault) staged: Final = _stage_token_file(_without_secret(stamped)) if isinstance(staged, CredentialNotSaved): return staged @@ -156,18 +156,42 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN return _keep_the_secret_in_the_file(stamped, outcome) -def _stamped_past_the_login_on_disk(record: CliTokenRecord) -> CliTokenRecord: - """Keep a sign-in's stamp ahead of the one it replaces, whatever the clock did in between. +def _stamped_past_every_stored_login(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord: + """Keep a sign-in's stamp ahead of every login already stored, whatever the clock did in between. The stamp is what decides a keychain secret against one still on disk, so a clock that stepped backwards between two logins would hand the older of them the win and put a superseded - credential back in use. The file already names the login being replaced, and pinning the new - stamp just past it costs one read that changes nothing on a clock that only moves forwards. + credential back in use. Pinning the new stamp just past the highest one either store holds costs + one read each and changes nothing on a clock that only moves forwards. + """ + highest: Final = _highest_stamp_already_stored(record.base_url, vault) + if highest < record.timestamp: + return record + return record.model_copy(update=MappingProxyType({"timestamp": math.nextafter(highest, math.inf)})) + + +def _highest_stamp_already_stored(base_url: str, vault: SecretVault) -> float: + """When the latest login either store still holds was made, or minus infinity when neither has one. + + Both are asked because the file names the login being replaced only while the two agree. A login + the keychain took but the file could not record afterwards leaves the keychain holding the later + of the two, and reading only the file would stamp the next sign-in below it. """ previous: Final = _read_token_file() - if previous is None or previous.timestamp < record.timestamp: - return record - return record.model_copy(update=MappingProxyType({"timestamp": math.nextafter(previous.timestamp, math.inf)})) + secret: Final = _stored_secret(base_url, vault) + return max( + -math.inf if previous is None else previous.timestamp, + -math.inf if secret is None else secret.timestamp, + ) + + +def _stored_secret(base_url: str, vault: SecretVault) -> CliTokenSecret | None: + """The keychain's secret for this server, when it holds one this login may be compared against""" + match vault.read(): + case SecretFound(blob=blob): + return _decode_secret(blob, base_url) + case SecretMissing() | KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): + return None def _keep_the_secret_in_the_file(record: CliTokenRecord, outcome: SecretWrite) -> SecretSave: diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index d9fc17f908f..e78add61dd5 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -528,6 +528,19 @@ class TestSaveCliToken: assert json.loads(vault.blob)["timestamp"] == 2000.0 assert json.loads(_token_file(isolated_home).read_text())["timestamp"] == 2000.0 + def test_a_login_is_stamped_past_the_keychain_the_file_could_not_keep_up_with( + self, isolated_home, secret_vault_factory + ): + """A login reported as CredentialNotRecorded leaves the keychain holding a later sign-in + than the file names, so the file alone is no longer the floor. A later login on a clock + that went back past that keychain entry still has to be the one served.""" + _write_legacy_file(isolated_home, key="sk-superseded", timestamp=1000.0) + vault = secret_vault_factory(blob=_blob(key="sk-recorded", timestamp=2000.0), writable=False) + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-fresh", timestamp=1500.0), vault=vault) + + assert load_cli_token(vault=vault).key == "sk-fresh" + class TestScrubFailure: """A keychain that took the secret while the file kept it is the worst of both stores: the From 1f2baf509e1d683e53a11daec6ac673d4dec9d52 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 05:28:19 -0700 Subject: [PATCH 26/53] test(cli): model keyring's null backend in the vault test double FakeSecretVault could only stand in for a discarding backend by passing KeyringDiscardsWrites as its `failure`, which also made read() and erase() hand it back. Neither SecretRead nor SecretErase admits that outcome and the real KeyringVault never produces it there, so the login path's match was falling through on a value it can never see. Give the double a `discards` flag that reports it from write() alone, which is what the null backend does. Also widen lint-format-check-changed's pathspec. Git wildmatch runs without FNM_PATHNAME here, so 'litellm/**/*.py' still requires an intermediate directory and silently skipped all 21 top-level modules, litellm/__init__.py and litellm/main.py among them. All 21 already pass ruff format. --- Makefile | 2 +- tests/test_litellm/conftest.py | 8 +++++++- tests/test_litellm/proxy/client/cli/test_auth_commands.py | 3 +-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ab6eba880bf..a47b3f0f66a 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ lint-install: # only the litellm Python files changed vs the base are checked, so a pre-existing # format issue elsewhere doesn't block an unrelated commit. lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) - @files=$$(git diff --name-only --diff-filter=ACMR origin/litellm_internal_staging...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' || true); \ + @files=$$(git diff --name-only --diff-filter=ACMR origin/litellm_internal_staging...HEAD -- 'litellm/*.py' | grep -v '^litellm/enterprise/' || true); \ if [ -z "$$files" ]; then \ echo "No changed litellm Python files to format-check."; \ else \ diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index b42355fa045..1229642dea0 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -23,6 +23,7 @@ from litellm import router as litellm_router_module from litellm import utils as litellm_utils_module from litellm._logging import ALL_LOGGERS from litellm.litellm_core_utils.cli_keyring import ( + KeyringDiscardsWrites, KeyringUnreachable, KeyringUnusable, SecretErase, @@ -132,7 +133,8 @@ class FakeSecretVault: `available=False` models a keychain that is locked or has no backend, `writable=False` one that refuses to store, `erasable=False` one that will not release what it already holds, and `failure` - picks which unusable state those report. + picks which unusable state those report. `discards=True` is keyring's null backend, which answers + reads and erases like any other yet keeps nothing it is given, so only writes report it. """ def __init__( @@ -142,12 +144,14 @@ class FakeSecretVault: available: bool = True, writable: bool = True, erasable: bool = True, + discards: bool = False, failure: KeyringUnusable = KeyringUnreachable(), ) -> None: self.blob: str | None = blob self.available: bool = available self.writable: bool = writable self.erasable: bool = erasable + self.discards: bool = discards self.failure: KeyringUnusable = failure self.reads: int = 0 self.writes: list[str] = [] @@ -163,6 +167,8 @@ class FakeSecretVault: self.writes.append(blob) if not (self.available and self.writable): return self.failure + if self.discards: + return KeyringDiscardsWrites() self.blob = blob return SecretStored() diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 1dd8a7ead92..4f44514167c 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -16,7 +16,6 @@ from litellm.constants import CLI_JWT_EXPIRATION_HOURS from litellm.litellm_core_utils.cli_keyring import ( DISABLE_KEYRING_ENV_VAR, KeyringDisabled, - KeyringDiscardsWrites, KeyringNotInstalled, ) from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord, save_cli_token @@ -1068,7 +1067,7 @@ class TestKeychainBackedCommands: ): """A backend that accepts writes and stores nothing must not be reported as keychain storage, because the file is then told to drop the only remaining copy.""" - result = self._login(secret_vault_factory(available=False, failure=KeyringDiscardsWrites())) + result = self._login(secret_vault_factory(discards=True)) token_file = isolated_home / ".litellm" / "token.json" assert result.exit_code == 0 From 1fe06a1280f84b8b4e477d1fbaee3555c28489d8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:14:40 -0700 Subject: [PATCH 27/53] test(cli): pin the shared stamp's effect on the freshness shortcut The stamp both orders the two stores and drives is_cli_token_fresh, and nothing tied the two together, so a login that inherits a stamp from the future could stop being a deliberate trade without anything failing. Also corrects the lint-format-check-changed comment: git pathspecs match recursively, so the target checks a superset of the CI step rather than an identical set. --- Makefile | 6 ++++-- .../litellm_core_utils/test_cli_token_utils.py | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a47b3f0f66a..c3aa106cf3e 100644 --- a/Makefile +++ b/Makefile @@ -142,9 +142,11 @@ lint-install: $(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev $(UV_RUN) python scripts/prisma_generate_if_needed.py -# Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step: +# Diff-scoped format check, mirroring test-linting.yml's "Check ruff format" step: # only the litellm Python files changed vs the base are checked, so a pre-existing -# format issue elsewhere doesn't block an unrelated commit. +# format issue elsewhere doesn't block an unrelated commit. Git pathspecs match +# recursively, so 'litellm/*.py' covers nested modules and the top-level files that +# CI's 'litellm/**/*.py' skips, which makes this target a superset of the CI step. lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) @files=$$(git diff --name-only --diff-filter=ACMR origin/litellm_internal_staging...HEAD -- 'litellm/*.py' | grep -v '^litellm/enterprise/' || true); \ if [ -z "$$files" ]; then \ diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index e78add61dd5..357d3ae1b10 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -870,6 +870,15 @@ class TestIsCliTokenFresh: assert is_cli_token_fresh(almost, buffer_hours=0.1) is False + def test_a_stamp_left_in_the_future_keeps_reporting_fresh_until_the_clock_catches_up(self): + """The stamp both orders the two stores and drives this shortcut, so a store left stamped + ahead of the clock hands that stamp to the next sign-in and keeps it looking fresh past the + expiry the gateway will actually enforce. Pinning that here so the shared stamp cannot stop + being a deliberate trade without this failing first.""" + ahead = CliTokenRecord(timestamp=time.time() + CLI_JWT_EXPIRATION_HOURS * 3600) + + assert is_cli_token_fresh(ahead) is True + class _FakeKeyringModule: def __init__(self, stored=None, *, get_error=None, set_error=None, delete_error=None, discard=False): From 4fac88790dc07c427ff3b75ca7e63b538cf496a0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:45:24 -0700 Subject: [PATCH 28/53] fix(mistral): correct zai-glm-5-2 limits, add cached-input price and glm-5-2 alias Mistral's live /v1/models reports max_context_length 1048576 and capabilities.reasoning true for zai-glm-5-2, and its docs price cached input at $0.14/M. Without cache_read_input_token_cost LiteLLM billed every cached prompt token at $0, so a repeat request against a 21k-token cached prefix logged $0.0000135 instead of its real cost. Mistral also serves the model under the short glm-5-2 name, which had no cost map entry at all and therefore no pricing, so add it alongside. --- ...odel_prices_and_context_window_backup.json | 26 ++++- model_prices_and_context_window.json | 26 ++++- ...test_mistral_zai_glm_5_2_model_metadata.py | 103 ++++++++++++++++++ 3 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b73feae90d3..650415279cb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29046,16 +29046,36 @@ "supports_tool_choice": true }, "mistral/zai-glm-5-2": { + "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.4e-06, "litellm_provider": "mistral", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.mistral.ai/models/zai-glm-5-2", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "mistral/glm-5-2": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "mistral", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.mistral.ai/models/zai-glm-5-2", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b73feae90d3..650415279cb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29046,16 +29046,36 @@ "supports_tool_choice": true }, "mistral/zai-glm-5-2": { + "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.4e-06, "litellm_provider": "mistral", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.mistral.ai/models/zai-glm-5-2", "supports_assistant_prefill": true, "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "mistral/glm-5-2": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "mistral", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.mistral.ai/models/zai-glm-5-2", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true }, diff --git a/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py b/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py new file mode 100644 index 00000000000..ad1f3b06e15 --- /dev/null +++ b/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py @@ -0,0 +1,103 @@ +import json +from pathlib import Path + +import pytest + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.types.utils import PromptTokensDetailsWrapper, Usage +from litellm.utils import supports_prompt_caching, supports_reasoning + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +GLM_5_2_MODELS = ("mistral/zai-glm-5-2", "mistral/glm-5-2") + +INPUT_COST = 1.4e-06 +CACHED_INPUT_COST = 1.4e-07 +OUTPUT_COST = 4.4e-06 + + +def _load(path): + with open(path) as f: + return json.load(f) + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + """Force get_model_info to resolve against the in-repo cost map instead of the + remote one fetched at import time, which still carries the pre-merge pricing.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model", GLM_5_2_MODELS) +def test_zai_glm_5_2_specs(model): + info = _load(MAIN_PATH).get(model) + assert info is not None, f"{model} missing from model_prices_and_context_window.json" + + assert info["litellm_provider"] == "mistral" + assert info["mode"] == "chat" + + assert info["input_cost_per_token"] == INPUT_COST + assert info["output_cost_per_token"] == OUTPUT_COST + assert info["cache_read_input_token_cost"] == CACHED_INPUT_COST + + assert info["max_input_tokens"] == 1048576 + assert info["max_output_tokens"] == 131072 + assert info["max_tokens"] == 131072 + + assert info["supports_assistant_prefill"] is True + assert info["supports_function_calling"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_reasoning"] is True + assert info["supports_response_schema"] is True + assert info["supports_tool_choice"] is True + + routed_model, provider, _, _ = get_llm_provider(model=model) + assert routed_model == model.split("/", 1)[1] + assert provider == "mistral" + + +@pytest.mark.parametrize("model", GLM_5_2_MODELS) +def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model): + """Mistral advertises reasoning and prompt caching on this model, so the helpers + every caller checks before sending a request must say so too.""" + assert supports_reasoning(model=model) is True + assert supports_prompt_caching(model=model) is True + + info = litellm.get_model_info(model=model) + assert info["max_input_tokens"] == 1048576 + assert info["max_output_tokens"] == 131072 + + +@pytest.mark.parametrize("model", GLM_5_2_MODELS) +def test_cached_prompt_tokens_bill_at_the_cached_rate(local_model_cost_map, model): + """A cache hit reports its reused tokens under prompt_tokens_details, and those + tokens cost a tenth of the input rate, not the full rate and not nothing.""" + usage = Usage( + prompt_tokens=21010, + completion_tokens=100, + total_tokens=21110, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=20992), + ) + + prompt_cost, completion_cost = litellm.cost_per_token( + model=model, usage_object=usage, custom_llm_provider="mistral" + ) + + assert prompt_cost == pytest.approx(18 * INPUT_COST + 20992 * CACHED_INPUT_COST) + assert completion_cost == pytest.approx(100 * OUTPUT_COST) + + +@pytest.mark.parametrize("model", GLM_5_2_MODELS) +def test_backup_matches_main(model): + """Ensure the bundled (backup) cost map stays in sync with the canonical file.""" + main_cost = _load(MAIN_PATH) + backup_cost = _load(BACKUP_PATH) + + assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps" From e12833e6b41511ca991f1fe54900ff5c343c6902 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 20 Aug 2026 10:58:27 -0700 Subject: [PATCH 29/53] fix(ui): make dark-mode form controls visible (#37648) * fix(ui): make dark-mode form controls visible Two dark-mode defects left form controls without any visual boundary or fill. `--input` and `--border` share one value in `.dark`, oklch(0.309), which resolves to rgb(48,48,48). Against `--background` (33) that is a 15-step stroke, and against `--popover` (42) it collapses to 6 steps out of 255, so a control inside any dialog is effectively undrawn. The controls also use `bg-transparent`, so there is no fill cue either and only the placeholder text renders. Measured 1.09:1 against the dialog surface where WCAG 1.4.11 asks for 3.0:1 on the boundary of a user interface component. Splitting `--input` off at oklch(0.56) restores 3.07:1 without touching `--border`, which stays where it is because it draws decorative separators rather than control boundaries. 91 controls across 19 routes were measured at the collapsed value, every one with an identical stroke and surface, so a single token covers all of them. Separately, `@tailwindcss/forms` paints a white fill on every bare control. The block above already neutralises that for `combobox-chip-input`, but its audit covered `components/ui` only, and hand-rolled controls elsewhere still render white on a dark page: typed text lands at 1.11:1 and native selects at 2.19:1 on `/model-hub-table`, `/playground`, `/guardrails`, `/mcp-servers` and `/models-and-endpoints`. Tracking `--background` fixes those at 14.51:1 and 7.34:1. Light mode is unchanged by both. The token edit is scoped to `.dark`, and `--background` in `:root` is the same white the plugin was already painting, verified control-by-control on a dev server: backgrounds stay rgb(255,255,255) and ratios stay 20.13:1 and 4.84:1. * fix(ui): keep the combobox chip input transparent under the bare-control fill The new base rule matched at (0,2,1) while the combobox chip-input override sits at (0,1,0), so ComboboxChipsInput lost its transparent background and painted an opaque page-colored rectangle inside the chips container, which carries its own bg-transparent / dark:bg-input/30 fill. Folding the exclusions into one :not() list adds the chip input and drops the selector to (0,1,1). Every @tailwindcss/forms base selector is wrapped in :where(), so it lands at (0,0,1); (0,1,1) still outweighs it and bare inputs, textareas and selects keep the fill this PR gives them. --- ui/litellm-dashboard/src/app/globals.css | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/app/globals.css b/ui/litellm-dashboard/src/app/globals.css index 8fa7cde2d37..04a09693287 100644 --- a/ui/litellm-dashboard/src/app/globals.css +++ b/ui/litellm-dashboard/src/app/globals.css @@ -134,7 +134,7 @@ --warning: oklch(0.828 0.189 84.429); --info: oklch(0.707 0.165 254.624); --border: oklch(0.309 0 0); - --input: oklch(0.309 0 0); + --input: oklch(0.56 0 0); --ring: oklch(0.569 0 0); --chart-1: oklch(0.488 0.243 264.376); --chart-2: oklch(0.696 0.17 162.48); @@ -230,6 +230,13 @@ letter-spacing: inherit; } + /* Same plugin's white fill, on the hand-rolled controls outside components/ui that the audit + above did not cover. Tracks --background so light mode keeps the white it already painted. + Delete along with the plugin. */ + :is(input, textarea, select):not([type="checkbox"], [type="radio"], [data-slot="combobox-chip-input"]) { + background-color: var(--color-background); + } + button:not(:disabled), [role="button"]:not(:disabled) { cursor: pointer; From c794dcb91d449a825106b0f8f3c33ffd93b2a4d5 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 20 Aug 2026 10:58:32 -0700 Subject: [PATCH 30/53] fix(ui): give status colours a readable foreground and drop the muted 70% step (#37649) * fix(ui): give status colours a readable foreground and drop the muted 70% step The four status tokens are lightened for dark mode, which is correct when they are used as text and wrong for the 27 places that use them as a background under `text-white`. Every one of those passes in light and fails in dark: success 1.78:1, warning 1.72:1, info 2.64:1, destructive 2.89:1. The cause is not 27 authoring mistakes, it is that no `--success-foreground` and no sibling ever existed, so `text-white` was the only thing available to write. Adding the four companions and registering them in `@theme` makes the correct pairing expressible, and the call sites then read `text-success-foreground` instead of a hardcoded colour. Dark lands at 9.98, 10.31, 6.72 and 6.15. Light is deliberately pure white rather than the near-white the other `-foreground` tokens use, so the four ratios stay at exactly the 4.95, 5.03, 5.25 and 4.77 they are today instead of drifting down to 4.73, 4.81, 5.02 and 4.56. Separately `text-muted-foreground/70` measures 2.75:1 on a light page and 4.31:1 on a dark one, so the same 183 occurrences fail AA in light and sit under it in dark. Dropping the opacity step takes them to 4.84:1 and 7.34:1. The identical step on the placeholder base rule goes with them, which is what put every input's placeholder at 2.75:1 in light. Residual, not addressed here: `text-muted-foreground` over `bg-muted` reaches 4.39:1 in light, still short of 4.5. Closing that needs `--muted-foreground` itself to move, which changes every secondary label in the product and is a design call rather than a defect fix. * fix(ui): finish the status-foreground swap and repoint no-op muted hovers Four sites still forced text-white on a status fill because the class sat on a child element rather than on the filled container, so the earlier sweep did not reach them. The compliance quick-test bubble was worse: it paired bg-info with text-success-foreground and its paragraph kept text-white on top, so the dark-theme contrast the PR set out to fix was still reachable there Dropping the /70 step also turned 21 existing "text-muted-foreground/70 hover:text-muted-foreground" pairs into hovers that change nothing, which local/no-noop-hover-variant flags as an error. The affordance was "brighten on hover", so these now hover to text-foreground, matching the 74 places that already spell it that way The remaining churn is prettier reflowing the handful of lines whose length changed, since the token names are longer than text-white * fix(ui): let the approve/reject confirm button pick the token its fill uses Both submission review dialogs put text-success-foreground on the shared button class while the fill below it swings between bg-success for Approve and bg-destructive for Reject, so Reject drew a success token over a destructive fill. The two tokens resolve to the same value today, so nothing looks wrong, but the pairing only holds by coincidence and would break the moment either token moves. Moving the token into the branch makes it track the fill * fix(ui): drop the last 70% placeholders, still live on the legacy utility Four inputs spell their placeholder colour with Tailwind's older placeholder- utility rather than placeholder:text-, so the sweep that dropped the 70% step passed over them. Tailwind 4.3 still emits that utility, and utilities sit after base in the layer order, so those four kept overriding the new input::placeholder rule and kept rendering at 70% in dark mode, which is the contrast failure this PR set out to close They now spell it the same way as the three placeholders the PR already converted, which both removes the step and settles on one spelling --- .../_components/CacheLeakageCard.tsx | 4 +- .../_components/TierTurnsChart.tsx | 2 +- .../_components/cost_tracking_settings.tsx | 4 +- .../pricing_calculator/multi_cost_results.tsx | 12 +-- .../_components/provider_margin_table.tsx | 2 +- .../_components/GuardrailsOverview.tsx | 2 +- .../_components/TeamGuardrailsTab.tsx | 12 +-- .../_components/add_guardrail_form.tsx | 2 +- .../_components/EnvVarsSection.tsx | 2 +- .../_components/MCPSubmissionsTab.tsx | 44 ++++++----- .../mcp-servers/_components/mcp_connect.tsx | 2 +- .../components/chat_ui/A2AMetrics.tsx | 8 +- .../chat_ui/AdditionalModelSettings.tsx | 10 +-- .../components/chat_ui/AgentBuilderView.tsx | 2 +- .../components/chat_ui/ChatImageUpload.tsx | 2 +- .../playground/components/chat_ui/ChatUI.tsx | 16 ++-- .../chat_ui/CodeInterpreterOutput.tsx | 4 +- .../chat_ui/CodeInterpreterTool.tsx | 2 +- .../components/chat_ui/FilePreviewCard.tsx | 4 +- .../components/chat_ui/RealtimePlayground.tsx | 10 ++- .../chat_ui/ResponsesImageUpload.tsx | 2 +- .../chat_ui/SearchResultsDisplay.tsx | 6 +- .../components/chat_ui/SessionManagement.tsx | 2 +- .../components/compareUI/CompareUI.tsx | 4 +- .../compareUI/components/ComparisonPanel.tsx | 2 +- .../components/complianceUI/ComplianceUI.tsx | 76 +++++++++---------- .../_components/ai_suggestion_modal.tsx | 12 +-- .../prompts/_components/prompt_info.tsx | 2 +- .../_components/general_settings.tsx | 2 +- .../components/EndpointUsageTable.tsx | 2 +- .../components/EntityUsage/EntityUsage.tsx | 6 +- .../EntityUsage/SpendByProvider.tsx | 4 +- .../components/UsageAIChatPanel.tsx | 10 +-- .../_components/components/UsagePageView.tsx | 14 ++-- ui/litellm-dashboard/src/app/globals.css | 14 +++- .../AIHub/UsefulLinksManagement.tsx | 4 +- .../GuardrailsMonitor/LogViewer.tsx | 6 +- .../GuardrailsMonitor/MetricCard.tsx | 2 +- .../src/components/HelpLink.tsx | 4 +- .../Navbar/UserDropdown/UserDropdown.tsx | 2 +- .../SSOSettings/RoleMappings.tsx | 4 +- .../Fallbacks/FallbackGroupConfig.tsx | 6 +- .../RouterSettings/Fallbacks/Fallbacks.tsx | 2 +- .../components/EntityUsage/TopKeyView.tsx | 10 +-- .../VirtualKeysPage/keyTableColumns.tsx | 4 +- .../src/components/activity_metrics.tsx | 2 +- .../add_model/ClassificationMethodConfig.tsx | 6 +- .../add_model/ComplexityRouterConfig.tsx | 6 +- .../add_model/EscalationKeywords.tsx | 2 +- .../components/add_model/KeywordTierRules.tsx | 2 +- .../add_model/SemanticKeywordMatching.tsx | 2 +- .../components/bulk_create_users_button.tsx | 14 +++- .../src/components/chat/ConversationList.tsx | 2 +- .../components/chat_ui/MCPEventsDisplay.tsx | 4 +- .../common_components/AutoRotationView.tsx | 4 +- .../RouterSettingsSummary.tsx | 4 +- .../BudgetFallbacksEditor.tsx | 4 +- .../src/components/logging_settings_view.tsx | 4 +- .../mcp_tools/ByokCredentialModal.tsx | 8 +- .../HealthChecksTableColumns.tsx | 2 +- .../components/model_group_alias_settings.tsx | 2 +- .../permissions/AgentPermissions.tsx | 2 +- .../permissions/MCPServerPermissions.tsx | 10 +-- .../permissions/VectorStorePermissions.tsx | 2 +- .../src/components/public_model_hub.tsx | 16 ++-- .../shared/advanced_date_picker.tsx | 2 +- .../src/components/shared/chart_loader.tsx | 2 +- .../src/components/team/TeamInfo.tsx | 8 +- .../components/templates/KeyInfoHeader.tsx | 2 +- .../components/templates/key_info_view.tsx | 4 +- .../GuardrailViewer/CompliancePanel.tsx | 4 +- .../GuardrailViewer/GuardrailViewer.tsx | 4 +- .../LogDetailsDrawer/LogDetailsDrawer.tsx | 2 +- .../view_logs/RequestLogsTableColumns.tsx | 4 +- .../src/components/view_logs/TypeBadges.tsx | 2 +- 75 files changed, 248 insertions(+), 232 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index c0b4150b4f4..ca47b71725d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -40,7 +40,7 @@ const compareRows = (a: CacheLeakageRow, b: CacheLeakageRow, sort: SortState): n const InfoTooltip = ({ info }: { info: string }) => ( }> - + {info} @@ -72,7 +72,7 @@ const SortableHead = ({ className="inline-flex items-center gap-1 font-medium hover:text-foreground" > {label} - + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx index 5b9b8563baa..e55ebc07656 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx @@ -133,7 +133,7 @@ const TierTurnsChart: React.FC = ({ view, autoRouters }) => {slice.tier} {Math.round((100 * slice.turns) / total).toLocaleString()}%

{slice.models.length > 0 && ( -

{slice.models.join(", ")}

+

{slice.models.join(", ")}

)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx index fa4e8daf9d0..3fce928ed67 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx @@ -223,7 +223,7 @@ const CostTrackingSettings: React.FC = ({ userID, use ) : (
= ({ userID, use ) : (
+
Token Pricing:{" "} {result.input_cost_per_token && ( Input ${formatNumberWithCommas(result.input_cost_per_token * 1_000_000, 2)}/1M @@ -294,17 +294,17 @@ const MultiCostResults: React.FC = ({ multiResult, timePe {record.error ? ( - - + - ) : ( {formatCost(record.cost_per_request)} )} {record.error ? ( - - + - ) : ( 0 ? "text-warning" : "text-muted-foreground/70"}`} + className={`font-mono text-sm ${(record.margin_cost_per_request ?? 0) > 0 ? "text-warning" : "text-muted-foreground"}`} > {formatCost(record.margin_cost_per_request)} @@ -312,7 +312,7 @@ const MultiCostResults: React.FC = ({ multiResult, timePe {record.error ? ( - - + - ) : ( {formatCost(periodCost)} )} @@ -325,7 +325,7 @@ const MultiCostResults: React.FC = ({ multiResult, timePe aria-expanded={isExpanded} aria-label={`${isExpanded ? "Hide" : "Show"} cost breakdown for ${record.model}`} onClick={() => toggleExpanded(record.id)} - className="text-muted-foreground/70 hover:text-muted-foreground" + className="text-muted-foreground hover:text-foreground" > {isExpanded ? : } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx index fa322d1b4d1..04823ac4aa0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx @@ -138,7 +138,7 @@ const ProviderMarginTable: React.FC = ({ autoFocus /> % - + + + $ 150 ? "text-destructive" : row.original.avgLatency > 50 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx index cc2d2051a24..c9de758f50d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx @@ -358,7 +358,7 @@ function GuardrailCard({ @@ -727,7 +727,7 @@ function DetailPanel({
@@ -218,7 +220,7 @@ function SubmissionRulesPanel({ requiredFields, onChange, onSave, isSaving }: Su
{field.label}
-
{field.description}
+
{field.description}
); @@ -235,7 +237,7 @@ function SubmissionRulesPanel({ requiredFields, onChange, onSave, isSaving }: Su await onSave(); setExpanded(false); }} - className="px-4 py-1.5 text-sm font-medium text-white bg-info hover:bg-info/80 disabled:opacity-50 rounded-md transition-colors" + className="px-4 py-1.5 text-sm font-medium text-info-foreground bg-info hover:bg-info/80 disabled:opacity-50 rounded-md transition-colors" > {isSaving ? "Savingā€¦" : "Save Rules"} @@ -296,11 +298,11 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe )} {server.url && (
- + {server.url}
)} -
+
Transport: {server.transport ?? "sse"} @@ -322,7 +324,7 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe @@ -341,7 +343,7 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe @@ -367,7 +369,11 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe allPassed ? "bg-success" : "bg-destructive" }`} > - {allPassed ? : } + {allPassed ? ( + + ) : ( + + )}
@@ -383,7 +389,7 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe @@ -392,7 +398,7 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe @@ -567,13 +573,13 @@ export function MCPSubmissionsTab({ accessToken }: MCPSubmissionsTabProps) {
- + setSearch(e.target.value)} - className="w-full pl-9 pr-4 py-2 border border-border rounded-md text-sm text-foreground placeholder-muted-foreground/70 focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-info" + className="w-full pl-9 pr-4 py-2 border border-border rounded-md text-sm text-foreground placeholder:text-muted-foreground focus:outline-hidden focus:ring-1 focus:ring-ring focus:border-info" />
setSearchPrompt(e.target.value)} placeholder="Search prompts..." - className="w-full border border-border rounded-lg pl-8 pr-3 py-1.5 text-xs placeholder:text-muted-foreground/70 focus:outline-hidden focus:ring-2 focus:ring-blue-500/20 focus:border-info" + className="w-full border border-border rounded-lg pl-8 pr-3 py-1.5 text-xs placeholder:text-muted-foreground focus:outline-hidden focus:ring-2 focus:ring-blue-500/20 focus:border-info" />
@@ -883,7 +883,7 @@ export default function ComplianceUI({ > Select All - Ā· + Ā· @@ -985,7 +985,7 @@ export default function ComplianceUI({ Required columns:{" "} prompt,{" "} expected_result{" "} - (fail or pass) + (fail or pass)

Optional columns:{" "} @@ -1049,14 +1049,14 @@ export default function ComplianceUI({ className="w-full flex items-center gap-2 px-3 py-2.5 text-left bg-muted hover:bg-accent transition-colors rounded-lg border border-border" > {isExpanded ? ( - + ) : ( - + )}

{fw.name} - {fwPromptCount} prompts + {fwPromptCount} prompts
{fwSelectedCount > 0 && ( @@ -1092,9 +1092,9 @@ export default function ComplianceUI({ className="w-full flex items-center gap-1.5 px-2.5 py-2 text-left hover:bg-accent transition-colors" > {isCatExpanded ? ( - + ) : ( - + )} {category.name} - + {category.prompts.length} {selectedInCat > 0 && ( @@ -1118,7 +1118,7 @@ export default function ComplianceUI({ {isCatExpanded && (
-

+

{category.description}

) : ( -

+

No policies or guardrails selected ā€” select above to test against specific rules.

)} @@ -1245,7 +1245,7 @@ export default function ComplianceUI({
- +

Type a prompt below to quickly test it.

@@ -1254,10 +1254,10 @@ export default function ComplianceUI({ {quickTestMessages.map((msg) => (

{msg.type === "system" && ( @@ -1284,7 +1284,7 @@ export default function ComplianceUI({ {isQuickTesting && (

- +
)} @@ -1300,23 +1300,23 @@ export default function ComplianceUI({ onKeyDown={handleQuickTestKeyDown} placeholder="Enter text to test..." rows={3} - className="w-full px-3 pt-3 pb-1 text-sm text-foreground placeholder:text-muted-foreground/70 focus:outline-hidden resize-none" + className="w-full px-3 pt-3 pb-1 text-sm text-foreground placeholder:text-muted-foreground focus:outline-hidden resize-none" />
- + Press Enter to submit Ā·{" "} Shift+Enter for new line - {quickTestInput.length} + {quickTestInput.length}