From 7e1f44f0cc15e639d8fe95fc8831c5a0b25cc76b Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 30 Jul 2026 02:38:28 +0000 Subject: [PATCH 001/226] feat(proxy): add admin toggle to block requests for models without pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/proxy/_types.py | 2 + litellm/proxy/auth/auth_checks.py | 33 +++++ .../cost_tracking_settings.py | 65 ++++++++ .../proxy/auth/test_auth_checks.py | 139 +++++++++++++++++- .../test_cost_tracking_settings.py | 56 +++++++ .../_components/cost_tracking_settings.tsx | 47 +++++- .../_components/use_block_unpriced_config.ts | 63 ++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 81 ++++++++++ 9 files changed, 482 insertions(+), 5 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/use_block_unpriced_config.ts diff --git a/litellm/__init__.py b/litellm/__init__.py index 3f8c742c5a2..d14f41ad49c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -443,6 +443,7 @@ max_end_user_budget_id: Optional[str] = None # backwards compatibility — arbitrary client-supplied identifiers still # pass through unchanged. validate_end_user_id_in_db: bool = False +block_requests_for_models_without_pricing: bool = False disable_end_user_cost_tracking: Optional[bool] = None disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c6d4ee1120a..450868edf06 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3540,6 +3540,8 @@ class ProxyErrorTypes(str, enum.Enum): Project does not have access to the model """ + model_cost_map_missing = "model_cost_map_missing" + expired_key = "expired_key" """ Key has expired diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c46bc110ca8..11a45e78410 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -274,6 +274,22 @@ def _is_cost_explicitly_configured(model: str, llm_router: "Router") -> bool: return False +def model_has_no_cost_mapping(model: Optional[str], llm_router: Optional[Router]) -> bool: + if not model or llm_router is None: + return False + + model_group_info = llm_router.get_model_group_info(model_group=model) + if model_group_info is None: + return False + + input_cost = model_group_info.input_cost_per_token or 0 + output_cost = model_group_info.output_cost_per_token or 0 + if input_cost > 0 or output_cost > 0: + return False + + return not _is_cost_explicitly_configured(model, llm_router) + + async def _run_project_checks( project_object: Optional[LiteLLM_ProjectTableCachedObj], _model: Optional[Union[str, List[str]]], @@ -534,6 +550,23 @@ async def common_checks( if route in MODEL_DISCOVERY_ROUTES: skip_budget_checks = True + if ( + litellm.block_requests_for_models_without_pricing + and isinstance(_model, str) + and RouteChecks.is_llm_api_route(route=route) + and model_has_no_cost_mapping(model=_model, llm_router=llm_router) + ): + raise ProxyException( + message=( + f"Model '{_model}' has no pricing in the cost map, so its spend would be tracked as $0. " + "Requests for unpriced models are blocked because 'block_requests_for_models_without_pricing' " + "is enabled. Add pricing for this model (input_cost_per_token/output_cost_per_token) to allow it." + ), + type=ProxyErrorTypes.model_cost_map_missing, + param="model", + code=status.HTTP_403_FORBIDDEN, + ) + # 1. If team is blocked if team_object is not None and team_object.blocked is True: raise Exception(f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if you're an admin.") diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index cd2c5704778..2c18de6b903 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -13,6 +13,7 @@ POST /cost/estimate - Estimate cost for a given model and token counts from typing import Dict, Optional, Tuple, Union from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger @@ -407,6 +408,70 @@ async def update_cost_margin_config( ) +class BlockUnpricedModelsRequest(BaseModel): + enabled: bool + + +class BlockUnpricedModelsResponse(BaseModel): + enabled: bool + + +@router.get( + "/config/block_requests_for_models_without_pricing", + tags=["Cost Tracking"], + dependencies=[Depends(user_api_key_auth)], + response_model=BlockUnpricedModelsResponse, +) +async def get_block_requests_for_models_without_pricing() -> BlockUnpricedModelsResponse: + return BlockUnpricedModelsResponse(enabled=bool(litellm.block_requests_for_models_without_pricing)) + + +@router.patch( + "/config/block_requests_for_models_without_pricing", + tags=["Cost Tracking"], + dependencies=[Depends(user_api_key_auth)], + response_model=BlockUnpricedModelsResponse, +) +async def update_block_requests_for_models_without_pricing( + request: BlockUnpricedModelsRequest, +) -> BlockUnpricedModelsResponse: + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_config, + store_model_in_db, + ) + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + if store_model_in_db is not True: + raise HTTPException( + status_code=500, + detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."}, + ) + + try: + config = await proxy_config.get_config() + if "litellm_settings" not in config: + config["litellm_settings"] = {} + config["litellm_settings"]["block_requests_for_models_without_pricing"] = request.enabled + await proxy_config.save_config(new_config=config) + + litellm.block_requests_for_models_without_pricing = request.enabled + verbose_proxy_logger.info(f"Updated block_requests_for_models_without_pricing: {request.enabled}") + + return BlockUnpricedModelsResponse(enabled=request.enabled) + except Exception as e: + verbose_proxy_logger.error(f"Error updating block_requests_for_models_without_pricing: {str(e)}") + raise HTTPException( + status_code=500, + detail={"error": f"Failed to update setting: {str(e)}"}, + ) + + @router.post( "/cost/estimate", tags=["Cost Tracking"], diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 34a353966bf..aec0ddc55f8 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -2,6 +2,7 @@ import asyncio import json import os import sys +from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch sys.path.insert( @@ -5164,4 +5165,140 @@ async def test_get_project_object_db_fetch_returns_cached_obj(): assert isinstance(result, LiteLLM_ProjectTableCachedObj) assert result.project_id == "p-1" - assert result.project_alias == "proj" + + +UNPRICED_UNDERLYING_MODEL = "openai/unpriced-model-lit4984-xyz" + + +def _router_with_priced_and_unpriced_models() -> "Router": + from litellm.router import Router + + return Router( + model_list=[ + { + "model_name": "priced-group", + "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}, + }, + { + "model_name": "unpriced-group", + "litellm_params": {"model": UNPRICED_UNDERLYING_MODEL, "api_key": "sk-test"}, + }, + ] + ) + + +def test_model_has_no_cost_mapping_priced_model_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + + router = _router_with_priced_and_unpriced_models() + + assert model_has_no_cost_mapping(model="priced-group", llm_router=router) is False + + +def test_model_has_no_cost_mapping_unpriced_model_is_true(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + + router = _router_with_priced_and_unpriced_models() + + assert model_has_no_cost_mapping(model="unpriced-group", llm_router=router) is True + + +def test_model_has_no_cost_mapping_no_model_or_router_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + + router = _router_with_priced_and_unpriced_models() + + assert model_has_no_cost_mapping(model=None, llm_router=router) is False + assert model_has_no_cost_mapping(model="unpriced-group", llm_router=None) is False + + +async def _run_common_checks( + model: Optional[str], llm_router: Optional["Router"], route: str = "/chat/completions" +) -> bool: + from fastapi import Request + + from litellm.proxy.auth.auth_checks import common_checks + + return await common_checks( + request_body={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route=route, + llm_router=llm_router, + proxy_logging_obj=MagicMock(), + valid_token=UserAPIKeyAuth(token="test-token"), + request=MagicMock(spec=Request), + ) + + +@pytest.mark.asyncio +async def test_common_checks_blocks_unpriced_model_when_enabled(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = _router_with_priced_and_unpriced_models() + + with pytest.raises(ProxyException) as exc_info: + await _run_common_checks(model="unpriced-group", llm_router=router) + + assert exc_info.value.code == "403" + assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing + assert exc_info.value.param == "model" + assert "unpriced-group" in exc_info.value.message + assert "pricing" in exc_info.value.message.lower() + + +@pytest.mark.asyncio +async def test_common_checks_allows_unpriced_model_when_disabled(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", False) + router = _router_with_priced_and_unpriced_models() + + result = await _run_common_checks(model="unpriced-group", llm_router=router) + + assert result is True + + +@pytest.mark.asyncio +async def test_common_checks_allows_priced_model_when_enabled(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = _router_with_priced_and_unpriced_models() + + result = await _run_common_checks(model="priced-group", llm_router=router) + + assert result is True + + +@pytest.mark.asyncio +async def test_common_checks_ignores_non_llm_route_when_enabled(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = _router_with_priced_and_unpriced_models() + + result = await _run_common_checks( + model="unpriced-group", llm_router=router, route="/model/new" + ) + + assert result is True + + +@pytest.mark.asyncio +async def test_common_checks_blocks_alias_resolving_to_unpriced_model(monkeypatch): + from litellm.router import Router + + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = Router( + model_list=[ + { + "model_name": "billed-underlying-group", + "litellm_params": {"model": UNPRICED_UNDERLYING_MODEL, "api_key": "sk-test"}, + } + ], + model_group_alias={"public-alias": "billed-underlying-group"}, + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_common_checks(model="public-alias", llm_router=router) + + assert exc_info.value.code == "403" + assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing + assert "public-alias" in exc_info.value.message diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index bc463d5e75d..4fb90e9fb2d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -500,3 +500,59 @@ class TestResolveModelForCostLookup: assert resolved_model == "openai/gpt-4" assert provider is None + + +class TestBlockRequestsForModelsWithoutPricing: + """Test suite for the block_requests_for_models_without_pricing toggle endpoints""" + + @pytest.mark.asyncio + async def test_get_reflects_in_memory_flag(self): + with patch.object(litellm, "block_requests_for_models_without_pricing", True): + response = client.get( + "/config/block_requests_for_models_without_pricing", + headers={"Authorization": "Bearer sk-1234"}, + ) + + assert response.status_code == 200 + assert response.json() == {"enabled": True} + + @pytest.mark.asyncio + async def test_patch_persists_and_updates_flag(self): + mock_proxy_config = AsyncMock() + mock_proxy_config.get_config = AsyncMock(return_value={"litellm_settings": {}}) + mock_proxy_config.save_config = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch.object(litellm, "block_requests_for_models_without_pricing", False), + ): + response = client.patch( + "/config/block_requests_for_models_without_pricing", + headers={"Authorization": "Bearer sk-1234"}, + json={"enabled": True}, + ) + + assert response.status_code == 200 + assert response.json() == {"enabled": True} + assert litellm.block_requests_for_models_without_pricing is True + + saved_config = mock_proxy_config.save_config.call_args.kwargs["new_config"] + assert saved_config["litellm_settings"]["block_requests_for_models_without_pricing"] is True + + @pytest.mark.asyncio + async def test_patch_requires_store_model_in_db(self): + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.proxy_config", AsyncMock()), + patch("litellm.proxy.proxy_server.store_model_in_db", False), + ): + response = client.patch( + "/config/block_requests_for_models_without_pricing", + headers={"Authorization": "Bearer sk-1234"}, + json={"enabled": True}, + ) + + assert response.status_code == 500 + assert "error" in response.json()["detail"] diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx index b32e7afd756..f6a3d487ade 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx @@ -12,7 +12,7 @@ import { TabPanels, TabPanel, } from "@tremor/react"; -import { Modal, Form } from "antd"; +import { Modal, Form, Switch } from "antd"; import { CostTrackingSettingsProps } from "./types"; import ProviderDiscountTable from "./provider_discount_table"; import AddProviderForm from "./add_provider_form"; @@ -24,6 +24,7 @@ import { DocsMenu } from "@/components/HelpLink"; import HowItWorks from "./how_it_works"; import { useDiscountConfig } from "./use_discount_config"; import { useMarginConfig } from "./use_margin_config"; +import { useBlockUnpricedConfig } from "./use_block_unpriced_config"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; const DOCS_LINKS = [ @@ -65,9 +66,16 @@ const CostTrackingSettings: React.FC = ({ userID, use handleMarginChange, } = useMarginConfig({ accessToken }); + const { + blockUnpriced, + isUpdating: isUpdatingBlockUnpriced, + fetchBlockUnpriced, + setBlockUnpriced, + } = useBlockUnpricedConfig({ accessToken }); + useEffect(() => { if (accessToken) { - Promise.all([fetchDiscountConfig(), fetchMarginConfig()]).finally(() => { + Promise.all([fetchDiscountConfig(), fetchMarginConfig(), fetchBlockUnpriced()]).finally(() => { setIsFetching(false); }); @@ -82,7 +90,7 @@ const CostTrackingSettings: React.FC = ({ userID, use }; loadModels(); } - }, [accessToken, fetchDiscountConfig, fetchMarginConfig]); + }, [accessToken, fetchDiscountConfig, fetchMarginConfig, fetchBlockUnpriced]); const handleAddProvider = async () => { const success = await addProvider(selectedProvider, newDiscount); @@ -293,7 +301,38 @@ const CostTrackingSettings: React.FC = ({ userID, use )} - {/* Accordion 3: Pricing Calculator - Available to all roles */} + {isProxyAdmin && ( + + +
+ Block Unpriced Models + + Reject requests for models that have no pricing in the cost map instead of logging them as $0 spend + +
+ + +
+
+
+ Block requests for models without pricing + + When enabled, a request whose resolved model has no cost mapping is rejected with a 403 so an + admin can add pricing for it. Off by default + +
+ setBlockUnpriced(checked)} + /> +
+
+ + + )} + + {/* Accordion 4: Pricing Calculator - Available to all roles */}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/use_block_unpriced_config.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/use_block_unpriced_config.ts new file mode 100644 index 00000000000..f4a110c7878 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/use_block_unpriced_config.ts @@ -0,0 +1,63 @@ +import { useState, useCallback } from "react"; +import { apiClient } from "@/components/networking"; +import NotificationsManager from "@/components/molecules/notifications_manager"; + +export interface UseBlockUnpricedConfigProps { + accessToken: string | null; +} + +export interface UseBlockUnpricedConfigReturn { + blockUnpriced: boolean; + isUpdating: boolean; + fetchBlockUnpriced: () => Promise; + setBlockUnpriced: (enabled: boolean) => Promise; +} + +interface BlockUnpricedResponse { + enabled: boolean; +} + +const ENDPOINT = "/config/block_requests_for_models_without_pricing"; + +export function useBlockUnpricedConfig({ accessToken }: UseBlockUnpricedConfigProps): UseBlockUnpricedConfigReturn { + const [blockUnpriced, setBlockUnpricedState] = useState(false); + const [isUpdating, setIsUpdating] = useState(false); + + const fetchBlockUnpriced = useCallback(async () => { + if (!accessToken) return; + try { + const data = await apiClient.get(ENDPOINT, { accessToken }); + setBlockUnpricedState(Boolean(data?.enabled)); + } catch (error) { + console.error("Error fetching block-unpriced-models setting:", error); + } + }, [accessToken]); + + const setBlockUnpriced = useCallback( + async (enabled: boolean) => { + if (!accessToken) return; + setIsUpdating(true); + try { + const data = await apiClient.patch(ENDPOINT, { accessToken, body: { enabled } }); + setBlockUnpricedState(Boolean(data?.enabled)); + NotificationsManager.success( + enabled + ? "Requests for models without pricing will now be blocked" + : "Requests for models without pricing are now allowed", + ); + } catch (error) { + console.error("Error updating block-unpriced-models setting:", error); + } finally { + setIsUpdating(false); + } + }, + [accessToken], + ); + + return { + blockUnpriced, + isUpdating, + fetchBlockUnpriced, + setBlockUnpriced, + }; +} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ed975c6be0a..5abe3b4d587 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1844,6 +1844,24 @@ export interface paths { patch?: never; trace?: never; }; + "/config/block_requests_for_models_without_pricing": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Block Requests For Models Without Pricing */ + get: operations["get_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Block Requests For Models Without Pricing */ + patch: operations["update_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_patch"]; + trace?: never; + }; "/config/callback/delete": { parameters: { query?: never; @@ -21247,6 +21265,16 @@ export interface components { /** Team Id */ team_id: string; }; + /** BlockUnpricedModelsRequest */ + BlockUnpricedModelsRequest: { + /** Enabled */ + enabled: boolean; + }; + /** BlockUnpricedModelsResponse */ + BlockUnpricedModelsResponse: { + /** Enabled */ + enabled: boolean; + }; /** BlockUsers */ BlockUsers: { /** User Ids */ @@ -37097,6 +37125,59 @@ export interface operations { }; }; }; + get_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BlockUnpricedModelsResponse"]; + }; + }; + }; + }; + update_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_patch: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BlockUnpricedModelsRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BlockUnpricedModelsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; delete_callback_config_callback_delete_post: { parameters: { query?: never; From 84c41dcdc3e55c180255433538f6df8969a56297 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 31 Jul 2026 03:02:10 +0000 Subject: [PATCH 002/226] fix(proxy): treat non-token pricing as priced and propagate the unpriced-model toggle across workers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/proxy/auth/auth_checks.py | 58 +++++++++++++++++-- .../proxy/auth/test_auth_checks.py | 45 ++++++++++++++ .../test_cost_tracking_settings.py | 14 +++++ 4 files changed, 112 insertions(+), 6 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 1014b472c61..b0ff992931f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1525,6 +1525,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [ "public_model_groups_links", "cost_discount_config", "cost_margin_config", + "block_requests_for_models_without_pricing", "budget_exceeded_throttle_percentage", # Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS) # must be listed here so a DB write from one worker overrides the live litellm attribute on diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 11a45e78410..136b8d19c04 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -13,7 +13,18 @@ import asyncio import math import re import time -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Mapping, + Optional, + Type, + Union, + cast, +) from fastapi import HTTPException, Request, status from pydantic import BaseModel @@ -274,17 +285,52 @@ def _is_cost_explicitly_configured(model: str, llm_router: "Router") -> bool: return False +def _has_positive_cost(value: object) -> bool: + if isinstance(value, bool): + return False + if isinstance(value, (int, float)): + return value > 0 + if isinstance(value, dict): + return any(_has_positive_cost(nested) for nested in value.values()) + return False + + +def _entry_has_priced_metric(entry: Mapping[str, object]) -> bool: + return any("cost_per" in key and _has_positive_cost(value) for key, value in entry.items()) + + +def _model_group_has_pricing(model: str, llm_router: "Router") -> bool: + """ + Check every deployment behind a model group for a positive price on any billed + metric (tokens, characters, seconds, pages, images, queries, ...), so models that + are billed by a non-token metric are not treated as unpriced. + """ + for deployment in llm_router.get_model_list(model_name=model) or []: + litellm_params = deployment.get("litellm_params") or {} + if _entry_has_priced_metric(litellm_params): + return True + + model_id = (deployment.get("model_info") or {}).get("id") + if model_id is None: + continue + + model_info = llm_router.get_deployment_model_info( + model_id=model_id, model_name=litellm_params.get("model") or "" + ) + if model_info is not None and _entry_has_priced_metric(model_info): + return True + + return False + + def model_has_no_cost_mapping(model: Optional[str], llm_router: Optional[Router]) -> bool: if not model or llm_router is None: return False - model_group_info = llm_router.get_model_group_info(model_group=model) - if model_group_info is None: + if llm_router.get_model_group_info(model_group=model) is None: return False - input_cost = model_group_info.input_cost_per_token or 0 - output_cost = model_group_info.output_cost_per_token or 0 - if input_cost > 0 or output_cost > 0: + if _model_group_has_pricing(model=model, llm_router=llm_router): return False return not _is_cost_explicitly_configured(model, llm_router) diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index aec0ddc55f8..a078e041aa7 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -5165,6 +5165,7 @@ async def test_get_project_object_db_fetch_returns_cached_obj(): assert isinstance(result, LiteLLM_ProjectTableCachedObj) assert result.project_id == "p-1" + assert result.project_alias == "proj" UNPRICED_UNDERLYING_MODEL = "openai/unpriced-model-lit4984-xyz" @@ -5212,6 +5213,50 @@ def test_model_has_no_cost_mapping_no_model_or_router_is_false(): assert model_has_no_cost_mapping(model="unpriced-group", llm_router=None) is False +@pytest.mark.parametrize( + "underlying_model", + [ + "azure/speech/azure-tts", + "mistral/mistral-ocr-latest", + "vertex_ai/imagen-3.0-generate-001", + ], +) +def test_model_has_no_cost_mapping_non_token_priced_model_is_false(underlying_model): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + from litellm.router import Router + + router = Router( + model_list=[ + { + "model_name": "non-token-priced-group", + "litellm_params": {"model": underlying_model, "api_key": "sk-test"}, + } + ] + ) + + assert model_has_no_cost_mapping(model="non-token-priced-group", llm_router=router) is False + + +def test_model_has_no_cost_mapping_non_token_price_from_litellm_params_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + from litellm.router import Router + + router = Router( + model_list=[ + { + "model_name": "custom-tts", + "litellm_params": { + "model": UNPRICED_UNDERLYING_MODEL, + "api_key": "sk-test", + "input_cost_per_second": 0.0001, + }, + } + ] + ) + + assert model_has_no_cost_mapping(model="custom-tts", llm_router=router) is False + + async def _run_common_checks( model: Optional[str], llm_router: Optional["Router"], route: str = "/chat/completions" ) -> bool: diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index 4fb90e9fb2d..1124a4a31d4 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -541,6 +541,20 @@ class TestBlockRequestsForModelsWithoutPricing: saved_config = mock_proxy_config.save_config.call_args.kwargs["new_config"] assert saved_config["litellm_settings"]["block_requests_for_models_without_pricing"] is True + def test_peer_workers_pick_up_persisted_flag_on_config_reload(self): + """A PATCH only mutates the flag on the worker that served it; peer workers must pick the + persisted value up when they reload litellm_settings from the DB.""" + from litellm.proxy.proxy_server import ProxyConfig + + with patch.object(litellm, "block_requests_for_models_without_pricing", False): + ProxyConfig()._update_config_fields( + current_config={}, + param_name="litellm_settings", + db_param_value={"block_requests_for_models_without_pricing": True}, + ) + + assert litellm.block_requests_for_models_without_pricing is True + @pytest.mark.asyncio async def test_patch_requires_store_model_in_db(self): with ( From 074b37b4f987f239725a165565be16ff5e1f9686 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 31 Jul 2026 03:08:29 +0000 Subject: [PATCH 003/226] refactor(proxy): flatten the pricing-metric check to avoid recursion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/auth_checks.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 136b8d19c04..3639ef245cf 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -285,18 +285,19 @@ def _is_cost_explicitly_configured(model: str, llm_router: "Router") -> bool: return False -def _has_positive_cost(value: object) -> bool: - if isinstance(value, bool): - return False - if isinstance(value, (int, float)): - return value > 0 - if isinstance(value, dict): - return any(_has_positive_cost(nested) for nested in value.values()) - return False +def _is_positive_cost(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0 def _entry_has_priced_metric(entry: Mapping[str, object]) -> bool: - return any("cost_per" in key and _has_positive_cost(value) for key, value in entry.items()) + for key, value in entry.items(): + if "cost_per" not in key: + continue + if _is_positive_cost(value): + return True + if isinstance(value, dict) and any(_is_positive_cost(nested) for nested in value.values()): + return True + return False def _model_group_has_pricing(model: str, llm_router: "Router") -> bool: From 658c67c1523a54d8d2e51ad848af4df3edd40b40 Mon Sep 17 00:00:00 2001 From: Shifat Islam Santo Date: Fri, 14 Aug 2026 14:19:48 -0500 Subject: [PATCH 004/226] fix: preserve prompt cache for mid-conversation system on unflagged Claude models --- .../messages/transformation.py | 56 +++++++++++++------ ...st_messages_mid_conversation_system_e2e.py | 6 +- ...onversation_system_native_providers_e2e.py | 16 +++--- ...azure_anthropic_messages_transformation.py | 20 +++++-- .../test_anthropic_claude3_transformation.py | 55 +++++++++++++----- ...artner_models_anthropic_messages_config.py | 29 +++++++++- 6 files changed, 130 insertions(+), 52 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 4d3354c58b7..1e016cffb0e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -159,8 +159,22 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def _is_system_role_message(message: Any) -> bool: return isinstance(message, dict) and message.get("role") == "system" + _CONVERTED_SYSTEM_NOTE: Final = ( + "Operator note (not from the user): the following was originally a mid-conversation system-role reminder." + ) + + def _system_role_message_as_user(self, message: dict) -> dict: + return { + **message, + "role": "user", + "content": [ + {"type": "text", "text": self._CONVERTED_SYSTEM_NOTE}, + *self._as_system_content_blocks(message.get("content")), + ], + } + def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None: - """Move ``role: "system"`` entries out of ``messages`` per the Anthropic + """Normalize ``role: "system"`` entries in ``messages`` per the Anthropic ``/v1/messages`` contract, which the first-party API, Bedrock Invoke, Vertex, and Azure Foundry all enforce identically. @@ -173,9 +187,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): stay: hoisting one mutates the ``system`` prefix and invalidates the prompt cache for the whole message history. Older Claude models reject the role in every position ("role 'system' is not supported on this model"), - so without the flag every system entry is hoisted to keep the request from - 400-ing. Billing-header system blocks are stripped from the top-level - ``system`` field regardless of whether anything was hoisted. + so without the flag a mid-conversation entry is converted to a user turn + in place (prefixed with an operator note) rather than hoisted: hoisting + would mutate the ``system`` prefix and likewise collapse the cache, while + the in-place conversion keeps everything before it byte-identical. + Billing-header system blocks are stripped from the top-level ``system`` + field regardless of whether anything was hoisted. Subclasses whose upstream rejects the role opt in by calling this from their ``transform_anthropic_messages_request``; the first-party Anthropic @@ -185,21 +202,24 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages: Final = anthropic_messages_request.get("messages") if not isinstance(messages, list): return - if _supports_factory( - model=model, - custom_llm_provider=self.custom_llm_provider, - key="supports_mid_conversation_system", - ): - leading_count: Final = next( - (i for i, m in enumerate(messages) if not self._is_system_role_message(m)), - len(messages), + leading_count: Final = next( + (i for i, m in enumerate(messages) if not self._is_system_role_message(m)), + len(messages), + ) + hoisted: Final = messages[:leading_count] + remaining: Final = ( + messages[leading_count:] + if _supports_factory( + model=model, + custom_llm_provider=self.custom_llm_provider, + key="supports_mid_conversation_system", ) - hoisted = messages[:leading_count] - remaining = messages[leading_count:] - else: - hoisted = [m for m in messages if self._is_system_role_message(m)] - remaining = [m for m in messages if not self._is_system_role_message(m)] - if hoisted: + else [ + self._system_role_message_as_user(m) if self._is_system_role_message(m) else m + for m in messages[leading_count:] + ] + ) + if hoisted or remaining != messages: anthropic_messages_request["messages"] = remaining system_content: Final = [ block diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py index fff2109b0cf..e1d836108e0 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py @@ -6,7 +6,7 @@ and the 5 family) must keep a mid-conversation system reminder in place inside ``messages`` so the top-level ``system`` prefix stays byte-identical and the prompt cache written on turn one is read back in full on turn two. Models without the flag (Claude 4.7 and older) reject the role inside ``messages`` -outright, so the proxy must hoist the reminder into the top-level ``system`` +outright, so the proxy must convert the reminder to a user turn in place field and the call must still return a completion instead of a provider 400. The conversation shape mirrors what Claude Code sends mid-session: a cached @@ -201,7 +201,7 @@ class TestBedrockInvokeMidConversationSystem: "llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works", exercised_on=[], ) - def test_unflagged_model_hoists_system_reminder_and_succeeds( + def test_unflagged_model_converts_system_reminder_and_succeeds( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: model = _register_invoke_deployment( @@ -227,5 +227,5 @@ class TestBedrockInvokeMidConversationSystem: assert completion.text.strip(), ( f"{model}: conversation with a mid-conversation system reminder " f"returned no text; the reminder was forwarded in place to a model " - f"that rejects role 'system' inside messages instead of being hoisted" + f"that rejects role 'system' inside messages instead of being converted to a user turn" ) diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py index 35ed3dc881a..6eae0d66c59 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py @@ -7,13 +7,13 @@ accepted in place on Claude 4.8+/5 (200) but rejected on Claude 4.7 and older ("role 'system' is not supported on this model", 400), and a *leading* system entry is rejected on every model ("messages.0: use the top-level 'system' parameter"). This mirrors Bedrock Invoke (PRs #32578/#32831/#32882); the same -model-gated hoist now runs for these two providers (customer RCA gap #3). +model-gated normalization now runs for these two providers (customer RCA gap #3). Flagged models (``supports_mid_conversation_system`` in the cost map: Claude 4.8+ and the 5 family) must keep the reminder in ``messages`` so the top-level ``system`` prefix stays byte-identical and the prompt cache written on turn one is read back in full on turn two. Unflagged models (Claude 4.7 and older) must -have the reminder hoisted into the top-level ``system`` field so the call +have the reminder converted to a user turn in place so the call returns a completion instead of a provider 400. The conversation shape mirrors what Claude Code sends mid-session: a cached @@ -206,7 +206,7 @@ def _assert_flagged_model_keeps_cache( ) -def _assert_unflagged_model_hoists_and_succeeds( +def _assert_unflagged_model_converts_and_succeeds( client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody ) -> None: model = _register_deployment(client, resources, params) @@ -228,7 +228,7 @@ def _assert_unflagged_model_hoists_and_succeeds( assert completion.text.strip(), ( f"{model}: conversation with a mid-conversation system reminder returned " f"no text; the reminder was forwarded in place to a model that rejects " - f"role 'system' inside messages instead of being hoisted" + f"role 'system' inside messages instead of being converted to a user turn" ) @@ -250,10 +250,10 @@ class TestAzureFoundryMidConversationSystem: "llm.messages.azure_foundry.mid_conversation_system.nonstream.works", exercised_on=[], ) - def test_unflagged_model_hoists_system_reminder_and_succeeds( + def test_unflagged_model_converts_system_reminder_and_succeeds( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: - _assert_unflagged_model_hoists_and_succeeds( + _assert_unflagged_model_converts_and_succeeds( endpoints_client, resources, _azure_params(self.UNFLAGGED_MODEL) ) @@ -276,9 +276,9 @@ class TestVertexMidConversationSystem: "llm.messages.vertex.mid_conversation_system.nonstream.works", exercised_on=[], ) - def test_unflagged_model_hoists_system_reminder_and_succeeds( + def test_unflagged_model_converts_system_reminder_and_succeeds( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: - _assert_unflagged_model_hoists_and_succeeds( + _assert_unflagged_model_converts_and_succeeds( endpoints_client, resources, _vertex_params(self.UNFLAGGED_MODEL) ) diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index f6446b43fab..add1e9967db 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -425,7 +425,7 @@ class TestAzureAnthropicMidConversationSystem: {"type": "text", "text": "Cite sources."}, ] - def test_unsupported_model_hoists_mid_conversation_system(self, local_model_cost_map): + def test_unsupported_model_converts_mid_conversation_system_in_place(self, local_model_cost_map): messages = [ {"role": "user", "content": "read the file"}, {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, @@ -437,13 +437,23 @@ class TestAzureAnthropicMidConversationSystem: ) assert result["messages"] == [ {"role": "user", "content": "read the file"}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Operator note (not from the user): the following was " + "originally a mid-conversation system-role reminder." + ), + }, + {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + ], + }, {"role": "assistant", "content": "reading"}, {"role": "user", "content": "continue"}, ] - assert result["system"] == [ - {"type": "text", "text": "Base."}, - {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, - ] + assert result["system"] == [{"type": "text", "text": "Base."}] def test_azure_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index fd66667af64..84df706b722 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -2125,13 +2125,14 @@ def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cos ] -def test_bedrock_invoke_transform_hoists_mid_conversation_system_for_older_claude(local_model_cost_map): - """Regression test for Claude Code 400s on pre-Opus-4.8 Bedrock models: - Invoke rejects ``role: "system"`` in every position on Opus 4.7, Sonnet 4.6, - Haiku 4.5, etc. ("role 'system' is not supported on this model"), so on - models without ``supports_mid_conversation_system`` every system entry must - be hoisted into the top-level ``system`` field, mid-conversation ones - included.""" +def test_bedrock_invoke_transform_converts_mid_conversation_system_for_older_claude(local_model_cost_map): + """Invoke rejects ``role: "system"`` in every position on Opus 4.7, Sonnet + 4.6, Haiku 4.5, etc. ("role 'system' is not supported on this model"), but + hoisting a mid-conversation reminder into the top-level ``system`` field + mutates the cached prefix and reprocesses the whole history. On models + without ``supports_mid_conversation_system`` the reminder is converted to a + user turn in place instead: the request stays valid and a cache breakpoint + before the reminder still hits.""" from litellm.types.router import GenericLiteLLMParams cfg = AmazonAnthropicClaudeMessagesConfig() @@ -2156,19 +2157,30 @@ def test_bedrock_invoke_transform_hoists_mid_conversation_system_for_older_claud assert result["messages"] == [ {"role": "user", "content": "read the file"}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Operator note (not from the user): the following was " + "originally a mid-conversation system-role reminder." + ), + }, + {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + ], + }, {"role": "assistant", "content": "reading"}, {"role": "user", "content": "continue"}, ] - assert result["system"] == [ - {"type": "text", "text": "Base."}, - {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, - ] + assert result["system"] == [{"type": "text", "text": "Base."}] -def test_bedrock_invoke_transform_hoists_all_system_for_unmapped_model(local_model_cost_map): +def test_bedrock_invoke_transform_converts_system_for_unmapped_model(local_model_cost_map): """A model with no cost-map entry and no fallback-generalization rule gets - the hoist-everything behavior: the safe default is a mutated cache prefix, - never a provider 400 from forwarding a role the model may not accept.""" + the unsupported-model treatment: the safe default converts the reminder to + a user turn in place, never a provider 400 from forwarding a role the model + may not accept, and never a mutated cache prefix.""" from litellm.types.router import GenericLiteLLMParams cfg = AmazonAnthropicClaudeMessagesConfig() @@ -2189,10 +2201,23 @@ def test_bedrock_invoke_transform_hoists_all_system_for_unmapped_model(local_mod assert result["messages"] == [ {"role": "user", "content": "hi"}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Operator note (not from the user): the following was " + "originally a mid-conversation system-role reminder." + ), + }, + {"type": "text", "text": "mid-conversation reminder"}, + ], + }, {"role": "assistant", "content": "hello"}, {"role": "user", "content": "continue"}, ] - assert result["system"] == [{"type": "text", "text": "mid-conversation reminder"}] + assert "system" not in result def test_bedrock_invoke_transform_keeps_system_in_place_for_unmapped_future_claude(local_model_cost_map): diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 292bddf1274..ef7db337a74 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -622,7 +622,7 @@ class TestVertexAnthropicMidConversationSystem: {"type": "text", "text": "Cite sources."}, ] - def test_unsupported_model_hoists_mid_conversation_system(self, local_model_cost_map): + def test_unsupported_model_converts_mid_conversation_system_in_place(self, local_model_cost_map): messages = [ {"role": "user", "content": "read the file"}, {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, @@ -634,12 +634,35 @@ class TestVertexAnthropicMidConversationSystem: ) assert result["messages"] == [ {"role": "user", "content": "read the file"}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Operator note (not from the user): the following was " + "originally a mid-conversation system-role reminder." + ), + }, + {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + ], + }, {"role": "assistant", "content": "reading"}, {"role": "user", "content": "continue"}, ] + assert result["system"] == [{"type": "text", "text": "Base."}] + + def test_unsupported_model_still_hoists_leading_system_run(self, local_model_cost_map): + messages = [ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": "Cite sources."}, + {"role": "user", "content": "hi"}, + ] + result = _vertex_transform("claude-sonnet-4-6", messages) + assert result["messages"] == [{"role": "user", "content": "hi"}] assert result["system"] == [ - {"type": "text", "text": "Base."}, - {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + {"type": "text", "text": "You are terse."}, + {"type": "text", "text": "Cite sources."}, ] From 1b5e50727c62cdec0c06ab7c1f54006ad8c8f8a9 Mon Sep 17 00:00:00 2001 From: Shifat Islam Santo Date: Fri, 14 Aug 2026 14:40:10 -0500 Subject: [PATCH 005/226] fix: reuse block builder for lint budget, assert unflagged cache e2e --- .../messages/transformation.py | 6 ++-- ...st_messages_mid_conversation_system_e2e.py | 30 +++++++++++++------ ...onversation_system_native_providers_e2e.py | 28 ++++++++++++----- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 1e016cffb0e..84fb7a1f45e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -167,10 +167,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return { **message, "role": "user", - "content": [ - {"type": "text", "text": self._CONVERTED_SYSTEM_NOTE}, - *self._as_system_content_blocks(message.get("content")), - ], + "content": self._as_system_content_blocks(self._CONVERTED_SYSTEM_NOTE) + + self._as_system_content_blocks(message.get("content")), } def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None: diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py index e1d836108e0..bd59df959c4 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py @@ -208,24 +208,36 @@ class TestBedrockInvokeMidConversationSystem: endpoints_client, resources, UNFLAGGED_INVOKE_MODEL ) key = resources.key(models=[model]) + system_block = _cacheable_system_block(unique_marker()) - body = RichMessagesRequest( + primed = _prime_prompt_cache(endpoints_client, key, model, system_block) + + reminder_turn_body = RichMessagesRequest( model=model, - system=[TextBlock(text="You are terse.")], + system=[system_block], messages=[ - _user_turn(f"Say hi. Run {unique_marker()}."), + _user_turn(primed.first_user_text, cached=True), _system_reminder_turn(), - RichMessage(role="assistant", content=[TextBlock(text="Hi.")]), - _user_turn("Say bye."), + RichMessage(role="assistant", content=[TextBlock(text="OK.")]), + _user_turn("Reply with one word again.", cached=True), ], ) - completion = unwrap(_post_messages(endpoints_client, key, body)) + second = unwrap(_post_messages(endpoints_client, key, reminder_turn_body)) - assert completion.role == "assistant", ( - f"{model}: unexpected role {completion.role!r}" + assert second.role == "assistant", ( + f"{model}: unexpected role {second.role!r}" ) - assert completion.text.strip(), ( + assert second.text.strip(), ( f"{model}: conversation with a mid-conversation system reminder " f"returned no text; the reminder was forwarded in place to a model " f"that rejects role 'system' inside messages instead of being converted to a user turn" ) + assert second.usage.cache_read_input_tokens >= primed.full_prefix_tokens, ( + f"{model}: reminder turn read {second.usage.cache_read_input_tokens} " + f"cached tokens, expected at least the {primed.full_prefix_tokens} " + f"cached on turn one ({primed.prefix_read_tokens} system prefix + " + f"{primed.first_turn_creation_tokens} first user turn); the reminder " + f"was hoisted into the top-level system field instead of being " + f"converted to a user turn in place, mutating the cached prefix and " + f"re-billing the conversation at cache-write pricing" + ) diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py index 6eae0d66c59..bf4e662e02e 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py @@ -211,25 +211,37 @@ def _assert_unflagged_model_converts_and_succeeds( ) -> None: model = _register_deployment(client, resources, params) key = resources.key(models=[model]) + system_block = _cacheable_system_block(unique_marker()) - body = RichMessagesRequest( + primed = _prime_prompt_cache(client, key, model, system_block) + + reminder_turn_body = RichMessagesRequest( model=model, - system=[TextBlock(text="You are terse.")], + system=[system_block], messages=[ - _user_turn(f"Say hi. Run {unique_marker()}."), + _user_turn(primed.first_user_text, cached=True), _system_reminder_turn(), - RichMessage(role="assistant", content=[TextBlock(text="Hi.")]), - _user_turn("Say bye."), + RichMessage(role="assistant", content=[TextBlock(text="OK.")]), + _user_turn("Reply with one word again.", cached=True), ], ) - completion = unwrap(_post_messages(client, key, body)) + second = unwrap(_post_messages(client, key, reminder_turn_body)) - assert completion.role == "assistant", f"{model}: unexpected role {completion.role!r}" - assert completion.text.strip(), ( + assert second.role == "assistant", f"{model}: unexpected role {second.role!r}" + assert second.text.strip(), ( f"{model}: conversation with a mid-conversation system reminder returned " f"no text; the reminder was forwarded in place to a model that rejects " f"role 'system' inside messages instead of being converted to a user turn" ) + assert second.usage.cache_read_input_tokens >= primed.full_prefix_tokens, ( + f"{model}: reminder turn read {second.usage.cache_read_input_tokens} cached " + f"tokens, expected at least the {primed.full_prefix_tokens} cached on turn " + f"one ({primed.prefix_read_tokens} system prefix + " + f"{primed.first_turn_creation_tokens} first user turn); the reminder was " + f"hoisted into the top-level system field instead of being converted to a " + f"user turn in place, mutating the cached prefix and re-billing the " + f"conversation at cache-write pricing" + ) class TestAzureFoundryMidConversationSystem: From 4cd1c81a2dcd905112a7bc388afb5ed1412b8aa7 Mon Sep 17 00:00:00 2001 From: Shifat Islam Santo Date: Fri, 14 Aug 2026 14:53:11 -0500 Subject: [PATCH 006/226] fix: add supports_mid_conversation_system to bare first-party Claude cost-map keys --- ...odel_prices_and_context_window_backup.json | 5 +++ model_prices_and_context_window.json | 5 +++ ...erimental_pass_through_messages_handler.py | 37 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b288269b0a2..4c1b46f85ab 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12106,6 +12106,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12493,6 +12494,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12528,6 +12530,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12566,6 +12569,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -47684,6 +47688,7 @@ }, "source": "https://docs.claude.com/en/docs/about-claude/models/overview", "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b288269b0a2..4c1b46f85ab 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12106,6 +12106,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12493,6 +12494,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12528,6 +12530,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -12566,6 +12569,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -47684,6 +47688,7 @@ }, "source": "https://docs.claude.com/en/docs/about-claude/models/overview", "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index f11324ca376..91f5023496a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -960,3 +960,40 @@ def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypat assert result == "translated" assert translation_calls["count"] == 1 assert "config" not in captured + + +def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): + """Regional and provider-prefixed Claude 4.8+/5 entries carry + ``supports_mid_conversation_system``, but the bare first-party keys + (``claude-opus-4-8``) that a plain ``custom_llm_provider="anthropic"`` + lookup resolves were missed, so that lookup reports the capability as + unset. Every mapped first-party entry the fallback rule matches must + carry the flag.""" + import json + import os + import re + + import litellm + + cost_map_path = os.path.join( + os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json" + ) + with open(cost_map_path) as f: + cost_map = json.load(f) + rules = cost_map["fallback_generalizations"]["rules"] + rule_pattern = next( + (r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"), + None, + ) + assert rule_pattern is not None, "claude-mid-conversation-system rule not found in fallback_generalizations" + pattern = re.compile(rule_pattern, re.IGNORECASE) + missing = [ + key + for key, info in cost_map.items() + if isinstance(info, dict) + and info.get("litellm_provider") == "anthropic" + and "claude" in key + and pattern.search(key) + and info.get("supports_mid_conversation_system") is not True + ] + assert missing == [] From ffa37d05b7cf16c9874101f3c737da19b4154aca Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sun, 16 Aug 2026 14:28:50 -0400 Subject: [PATCH 007/226] feat(mistral): add zai-glm-5-2 model pricing and metadata --- .../model_prices_and_context_window_backup.json | 14 ++++++++++++++ model_prices_and_context_window.json | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e6c6cab0631..b73feae90d3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29045,6 +29045,20 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/zai-glm-5-2": { + "input_cost_per_token": 1.4e-06, + "litellm_provider": "mistral", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.mistral.ai/models/zai-glm-5-2", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "mistral/magistral-medium-2506": { "deprecation_date": "2025-11-30", "input_cost_per_token": 2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e6c6cab0631..b73feae90d3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29045,6 +29045,20 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "mistral/zai-glm-5-2": { + "input_cost_per_token": 1.4e-06, + "litellm_provider": "mistral", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.mistral.ai/models/zai-glm-5-2", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "mistral/magistral-medium-2506": { "deprecation_date": "2025-11-30", "input_cost_per_token": 2e-06, From 539a61be080b9ed8100ef00fca77f5b6175d8853 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sun, 16 Aug 2026 14:44:09 -0400 Subject: [PATCH 008/226] feat(perplexity): add Agent API third-party models (DeepSeek V4 Flash, GLM 5.2, Kimi K3, Kimi K2.7 Code) --- ...odel_prices_and_context_window_backup.json | 44 +++++++++++++++++++ model_prices_and_context_window.json | 44 +++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e6c6cab0631..b786a84ada7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34286,6 +34286,50 @@ "supports_reasoning": false, "supports_function_calling": true }, + "perplexity/perplexity/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 2.6e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 1.5e-05, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 4e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, "perplexity/pplx-embed-v1-0.6b": { "input_cost_per_token": 4e-09, "litellm_provider": "perplexity", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e6c6cab0631..b786a84ada7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34286,6 +34286,50 @@ "supports_reasoning": false, "supports_function_calling": true }, + "perplexity/perplexity/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 2.6e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 1.5e-05, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 4e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, "perplexity/pplx-embed-v1-0.6b": { "input_cost_per_token": 4e-09, "litellm_provider": "perplexity", From 782746553c109bdec6aa5ecddf8e674f5167f575 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sun, 16 Aug 2026 14:57:11 -0400 Subject: [PATCH 009/226] fix(perplexity): accept float usage.cost in cost_per_token, not just dict ResponseAPIUsage.parse_cost already flattens Perplexity's usage.cost.total_cost dict down to a float before it reaches the perplexity cost calculator, so the isinstance(cost_info, dict) check was always False on that path. Every Responses-mode Perplexity model was silently falling back to manual token-rate calculation and recording $0 spend whenever static per-token rates were missing. --- litellm/llms/perplexity/cost_calculator.py | 19 ++++++++------ .../test_perplexity_cost_calculator.py | 25 +++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 337fa8e630d..27835ecbfe8 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -21,14 +21,19 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ ## USE PRE-CALCULATED COST FROM PERPLEXITY IF AVAILABLE - ## Perplexity returns accurate cost in usage.cost.total_cost including request fees + ## Perplexity returns accurate cost in usage.cost.total_cost including request fees. + ## By the time it reaches here, ResponseAPIUsage.parse_cost has already flattened + ## that dict down to a float, so both shapes must be accepted. cost_info: Final = getattr(usage, "cost", None) - if cost_info is not None and isinstance(cost_info, dict): - total_cost: Final = cost_info.get("total_cost") - if total_cost is not None: - # Return total cost as completion_cost (prompt_cost=0) since Perplexity - # doesn't break down by input/output in their cost object - return (0.0, float(total_cost)) + total_cost: float | None = None + if isinstance(cost_info, dict): + total_cost = cost_info.get("total_cost") + elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool): + total_cost = float(cost_info) + if total_cost is not None: + # Return total cost as completion_cost (prompt_cost=0) since Perplexity + # doesn't break down by input/output in their cost object + return (0.0, float(total_cost)) ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it ## GET MODEL INFO diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index 46c1e457d7c..71ccb494cc7 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -400,6 +400,31 @@ class TestPerplexityCostCalculator: assert completion_cost == 0.008 assert prompt_cost + completion_cost == 0.008 + def test_uses_perplexity_provided_cost_when_normalized_to_float(self): + """ + Regression: for Responses API / Agent API models, `ResponseAPIUsage.parse_cost` + (litellm/types/llms/openai.py) already flattens Perplexity's + `usage.cost.total_cost` dict down to a plain float before + `_transform_response_api_usage_to_chat_usage` (litellm/responses/utils.py) copies + it onto the chat `Usage` object. So `usage.cost` arrives here as a float, not a + dict, on that path. + + Pre-fix, the `isinstance(cost_info, dict)` check was always False for a float, + so the pre-calculated cost branch was dead code for every Responses-mode + Perplexity model and it silently fell back to manual token-rate calculation, + recording $0 for any model missing static per-token rates (e.g. + perplexity/openai/gpt-5.2 before rates existed). + """ + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + usage.cost = 0.008 + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-pro", usage=usage + ) + + assert prompt_cost == 0.0 + assert completion_cost == 0.008 + def test_falls_back_to_manual_calculation_when_no_cost_provided(self): """ Test that manual cost calculation is used when Perplexity doesn't From 5d5dc4523fb950e131a235bea9a4f767ba7e0e17 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 19 Aug 2026 03:56:32 +0000 Subject: [PATCH 010/226] fix(cost): price streamed Messages usage via calculate_usage and the logging obj Streamed `/v1/messages` `usage.cost` disagreed with the cost the logging callback recorded in three ways: `input_tokens` was read as the whole prompt total, but Anthropic reports it excluding cache tokens, so the non-cached input went unbilled on cache hits; the `cache_creation` 5m/1h split was dropped, billing 1h writes at the 5m rate; and costing by model name alone ignored the deployment's custom pricing, so a negotiated discount still streamed sticker price. Anthropic usage now goes through `AnthropicConfig.calculate_usage`, the same transformation the non-streaming path uses, and the chunk is priced through the call's logging object when there is one so it inherits `custom_pricing`, `custom_llm_provider`, `base_model` and `router_model_id`, falling back to `completion_cost` by model name. `calculate_usage` only reads its `usage_object`, so it now takes a `Mapping`. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/chat/transformation.py | 2 +- litellm/proxy/common_request_processing.py | 130 ++++++++++------ .../streaming_handler.py | 2 +- .../proxy/test_common_request_processing.py | 140 ++++++++++++++++++ 4 files changed, 226 insertions(+), 48 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ef4ad7011c5..b4f040b0a8c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2152,7 +2152,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def calculate_usage( self, - usage_object: dict, + usage_object: Mapping[str, Any], reasoning_content: str | None, completion_response: dict | None = None, speed: str | None = None, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index a0b69ecb0bf..f5299b273b4 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -158,7 +158,7 @@ ProxyRouteType: TypeAlias = Literal[ "acancel_run", "adelete_run", ] -from litellm.types.utils import ServerToolUse +from litellm.llms.anthropic.chat.transformation import AnthropicConfig # Type alias for streaming chunk serializer (chunk after hooks + cost injection -> wire format) StreamChunkSerializer = Callable[[Any], str] @@ -3321,7 +3321,9 @@ class ProxyBaseLLMRequestProcessing: str_so_far += str(chunk.get("content", "")) model_name = request_data.get("model", "") - chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection(chunk, model_name) + chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection( + chunk, model_name, request_data.get("litellm_logging_obj") + ) # Set before the yield: an async generator suspends at the yield, # so a GeneratorExit on client disconnect is raised there and any @@ -3418,20 +3420,28 @@ class ProxyBaseLLMRequestProcessing: @overload @staticmethod - def _process_chunk_with_cost_injection(chunk: bytes, model_name: str) -> bytes: ... + def _process_chunk_with_cost_injection( + chunk: bytes, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None + ) -> bytes: ... @overload @staticmethod - def _process_chunk_with_cost_injection(chunk: object, model_name: str) -> object: ... + def _process_chunk_with_cost_injection( + chunk: object, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None + ) -> object: ... @staticmethod - def _process_chunk_with_cost_injection(chunk: object, model_name: str) -> object: + def _process_chunk_with_cost_injection( + chunk: object, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None + ) -> object: """ Process a streaming chunk and inject cost information if enabled. Args: chunk: The streaming chunk (dict, str, bytes, or bytearray) model_name: Model name for cost calculation + litellm_logging_obj: The call's logging object, used to price the chunk with + the same custom/deployment pricing as the logging callback Returns: The processed chunk with cost information injected if applicable @@ -3441,21 +3451,27 @@ class ProxyBaseLLMRequestProcessing: try: if isinstance(chunk, dict): - maybe_modified: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(chunk, model_name) + maybe_modified: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict( + chunk, model_name, litellm_logging_obj + ) if maybe_modified is not None: return maybe_modified elif isinstance(chunk, (bytes, bytearray)): try: s: Final = chunk.decode("utf-8") if s.endswith(("\n\n", "\r\n\r\n")): - maybe_mod = ProxyBaseLLMRequestProcessing._inject_cost_into_sse_frame_str(s, model_name) + maybe_mod = ProxyBaseLLMRequestProcessing._inject_cost_into_sse_frame_str( + s, model_name, litellm_logging_obj + ) if maybe_mod is not None: return maybe_mod.encode("utf-8") except Exception: pass elif isinstance(chunk, str): # Try to parse SSE frame and inject cost into the data line - maybe_mod = ProxyBaseLLMRequestProcessing._inject_cost_into_sse_frame_str(chunk, model_name) + maybe_mod = ProxyBaseLLMRequestProcessing._inject_cost_into_sse_frame_str( + chunk, model_name, litellm_logging_obj + ) if maybe_mod is not None: # Ensure trailing frame separator return maybe_mod if maybe_mod.endswith("\n\n") else (maybe_mod + "\n\n") @@ -3466,13 +3482,16 @@ class ProxyBaseLLMRequestProcessing: return chunk @staticmethod - def _inject_cost_into_sse_frame_str(frame_str: str, model_name: str) -> str | None: + def _inject_cost_into_sse_frame_str( + frame_str: str, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None + ) -> str | None: """ Inject cost information into an SSE frame string by modifying the JSON in the 'data:' line. Args: frame_str: SSE frame string that may contain multiple lines model_name: Model name for cost calculation + litellm_logging_obj: The call's logging object, forwarded for pricing Returns: Modified SSE frame string with cost injected, or None if no modification needed @@ -3486,7 +3505,9 @@ class ProxyBaseLLMRequestProcessing: json_part = stripped_ln.split("data:", 1)[1].strip() if json_part and json_part != "[DONE]": obj = json.loads(json_part) - maybe_modified = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(obj, model_name) + maybe_modified = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict( + obj, model_name, litellm_logging_obj + ) if maybe_modified is not None: lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "") return "\n".join(lines) @@ -3494,34 +3515,6 @@ class ProxyBaseLLMRequestProcessing: except Exception: return None - @staticmethod - def _anthropic_stream_usage_kwargs(usage: Mapping[str, Any]) -> Mapping[str, Any]: - prompt_tokens: Final = int(usage.get("input_tokens", 0) or 0) - completion_tokens: Final = int(usage.get("output_tokens", 0) or 0) - total_tokens: Final = int( - usage.get("total_tokens", prompt_tokens + completion_tokens) or (prompt_tokens + completion_tokens) - ) - web_search_requests: Final = usage.get("web_search_requests") - server_tool_use: Final = ( - ServerToolUse(web_search_requests=web_search_requests) if web_search_requests is not None else None - ) - return MappingProxyType( - { - key: value - for key, value in ( - ("prompt_tokens", prompt_tokens), - ("completion_tokens", completion_tokens), - ("total_tokens", total_tokens), - ("completion_tokens_details", usage.get("completion_tokens_details")), - ("prompt_tokens_details", usage.get("prompt_tokens_details")), - ("cache_creation_input_tokens", usage.get("cache_creation_input_tokens")), - ("cache_read_input_tokens", usage.get("cache_read_input_tokens")), - ("server_tool_use", server_tool_use), - ) - if value is not None - } - ) - @staticmethod def _openai_stream_usage_kwargs(usage: Mapping[str, Any]) -> Mapping[str, Any]: prompt_tokens: Final = int(usage.get("prompt_tokens", 0) or 0) @@ -3544,11 +3537,19 @@ class ProxyBaseLLMRequestProcessing: ) @staticmethod - def _stream_usage_kwargs_for_event(obj: Mapping[str, object], usage: Mapping[str, Any]) -> Mapping[str, Any] | None: + def _stream_usage_for_event(obj: Mapping[str, object], usage: Mapping[str, Any]) -> Usage | None: + """ + Build the ``Usage`` to price a streamed usage event. + + Anthropic goes through ``AnthropicConfig.calculate_usage``, the same transformation + the non-streaming path uses, so ``prompt_tokens`` is the full input total and the + cache read plus 5m/1h cache creation split land in ``prompt_tokens_details`` where + the pricer looks for them. + """ if obj.get("type") == "message_delta": - return ProxyBaseLLMRequestProcessing._anthropic_stream_usage_kwargs(usage) + return AnthropicConfig().calculate_usage(usage_object=usage, reasoning_content=None) if obj.get("object") == "chat.completion.chunk": - return ProxyBaseLLMRequestProcessing._openai_stream_usage_kwargs(usage) + return Usage(**ProxyBaseLLMRequestProcessing._openai_stream_usage_kwargs(usage)) return None @staticmethod @@ -3563,7 +3564,41 @@ class ProxyBaseLLMRequestProcessing: return None @staticmethod - def _inject_cost_into_usage_dict(obj: dict, model_name: str) -> dict | None: + def _logging_obj_cost_or_none( + model_response: ModelResponse, litellm_logging_obj: LiteLLMLoggingObj + ) -> float | None: + try: + cost: Final = litellm_logging_obj._response_cost_calculator(result=model_response) # pyright: ignore[reportPrivateUsage] # reuse the call's own cost calc for pricing parity with the logging callback + except Exception: + return None + return float(cost) if isinstance(cost, (int, float)) and not isinstance(cost, bool) else None + + @staticmethod + def _streamed_usage_cost( + model_response: ModelResponse, + model_name: str, + service_tier: str | None, + litellm_logging_obj: LiteLLMLoggingObj | None, + ) -> float | None: + """ + Price a streamed usage response through the call's logging object when there is one, + so the streamed cost picks up the same custom/deployment pricing (``custom_pricing``, + ``custom_llm_provider``, ``base_model``, ``router_model_id``) as the logging callback + rather than the model's sticker price; fall back to ``completion_cost`` by model name. + """ + cost_from_logging_obj: Final = ( + ProxyBaseLLMRequestProcessing._logging_obj_cost_or_none(model_response, litellm_logging_obj) + if litellm_logging_obj is not None + else None + ) + if cost_from_logging_obj is not None: + return cost_from_logging_obj + return ProxyBaseLLMRequestProcessing._completion_cost_or_none(model_response, model_name, service_tier) + + @staticmethod + def _inject_cost_into_usage_dict( + obj: dict, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None + ) -> dict | None: """ Inject cost information into the usage object of a streamed usage event (Anthropic ``message_delta`` or OpenAI ``chat.completion.chunk``). @@ -3571,6 +3606,8 @@ class ProxyBaseLLMRequestProcessing: Args: obj: Dictionary containing the SSE event data model_name: Model name for cost calculation + litellm_logging_obj: The call's logging object, used so the injected cost + matches the cost the logging callback records Returns: Modified dictionary with cost injected, or None if no modification needed @@ -3578,14 +3615,15 @@ class ProxyBaseLLMRequestProcessing: usage: Final = obj.get("usage") if not isinstance(usage, dict): return None - usage_kwargs: Final = ProxyBaseLLMRequestProcessing._stream_usage_kwargs_for_event(obj, usage) - if usage_kwargs is None: + stream_usage: Final = ProxyBaseLLMRequestProcessing._stream_usage_for_event(obj, usage) + if stream_usage is None: return None service_tier: Final = obj.get("service_tier") - cost_val: Final = ProxyBaseLLMRequestProcessing._completion_cost_or_none( - ModelResponse(usage=Usage(**usage_kwargs)), + cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost( + ModelResponse(usage=stream_usage), model_name, service_tier if isinstance(service_tier, str) else None, + litellm_logging_obj, ) if cost_val is None: return None diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 697eb7b96eb..b71622fc33d 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -106,7 +106,7 @@ class PassThroughStreamingHandler: ) # rebind-ok: SSE frame reassembly buffer across transport chunks if complete_frames: yield ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection( - complete_frames, resolved_model_name + complete_frames, resolved_model_name, litellm_logging_obj ) if pending: yield pending diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 355c6d27eb2..083982778b9 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6082,6 +6082,121 @@ class TestInjectCostIntoUsageDict: injected = json.loads(result.split("\n")[0].split("data:", 1)[1].strip()) assert injected["usage"]["cost"] == pytest.approx(self._expected_cost("gpt-4o-mini", 11, 4)) + def test_message_delta_cost_charges_the_non_cached_input_tokens(self): + """Anthropic reports ``input_tokens`` excluding cache tokens, so reading it as the whole + prompt total drops the non-cached input from the bill on every cache hit.""" + model = "claude-haiku-4-5" + pricing = litellm.model_cost[model] + event = { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": { + "input_tokens": 14, + "output_tokens": 8, + "cache_read_input_tokens": 3202, + "cache_creation_input_tokens": 0, + }, + } + + result = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, model) + + assert result is not None + expected = ( + 14 * pricing["input_cost_per_token"] + + 3202 * pricing["cache_read_input_token_cost"] + + 8 * pricing["output_cost_per_token"] + ) + dropped_input = expected - 14 * pricing["input_cost_per_token"] + assert result["usage"]["cost"] == pytest.approx(expected) + assert result["usage"]["cost"] > dropped_input + + def test_message_delta_prices_1h_cache_creation_above_the_5m_rate(self): + """The ``cache_creation`` 5m/1h split has to survive into ``prompt_tokens_details``, + otherwise a 1h write is billed at the cheaper 5m rate.""" + model = "claude-haiku-4-5" + pricing = litellm.model_cost[model] + event = { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": { + "input_tokens": 14, + "output_tokens": 8, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 2000, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2000}, + }, + } + + result = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, model) + + assert result is not None + base = 14 * pricing["input_cost_per_token"] + 8 * pricing["output_cost_per_token"] + expected_1h = base + 2000 * pricing["cache_creation_input_token_cost_above_1hr"] + flat_5m = base + 2000 * pricing["cache_creation_input_token_cost"] + assert expected_1h != pytest.approx(flat_5m) + assert result["usage"]["cost"] == pytest.approx(expected_1h) + + def test_message_delta_prices_through_the_logging_obj_so_custom_pricing_applies(self): + """Costing by model name alone yields sticker price, so a deployment with a negotiated + discount streamed a ``usage.cost`` that disagreed with the callback's ``response_cost``.""" + + class _StubLoggingObj: + def __init__(self, cost): + self._cost = cost + self.captured_result = None + + def _response_cost_calculator(self, result): + self.captured_result = result + return self._cost + + model = "claude-haiku-4-5" + discounted_cost = 0.00099 + stub = _StubLoggingObj(discounted_cost) + event = { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": { + "input_tokens": 14, + "output_tokens": 8, + "cache_read_input_tokens": 3202, + "cache_creation_input_tokens": 500, + "cache_creation": {"ephemeral_5m_input_tokens": 100, "ephemeral_1h_input_tokens": 400}, + }, + } + + result = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, model, stub) + + assert result is not None + assert result["usage"]["cost"] == discounted_cost + assert result["usage"]["cost"] != pytest.approx(self._expected_cost(model, 14 + 500 + 3202, 8)) + usage = stub.captured_result.usage + assert usage.prompt_tokens == 14 + 500 + 3202 + details = usage.prompt_tokens_details.cache_creation_token_details + assert details.ephemeral_5m_input_tokens == 100 + assert details.ephemeral_1h_input_tokens == 400 + + def test_message_delta_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self): + class _StubLoggingObj: + def _response_cost_calculator(self, result): + return None + + model = "claude-haiku-4-5" + pricing = litellm.model_cost[model] + event = { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"input_tokens": 14, "output_tokens": 8, "cache_read_input_tokens": 3202}, + } + + result = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, model, _StubLoggingObj()) + + assert result is not None + assert result["usage"]["cost"] == pytest.approx( + 14 * pricing["input_cost_per_token"] + + 3202 * pricing["cache_read_input_token_cost"] + + 8 * pricing["output_cost_per_token"] + ) + class TestProcessChunkWithCostInjection: def test_complete_usage_frame_chunk_is_injected(self, monkeypatch): @@ -6116,6 +6231,31 @@ class TestProcessChunkWithCostInjection: assert ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection(chunk, "gpt-4o-mini") == chunk + def test_message_delta_frame_is_priced_with_the_logging_obj(self, monkeypatch): + """Pins that the logging object reaches the pricer through the byte-frame entry point, + which is how the proxy actually calls this on a streamed Messages API request.""" + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) + + class _StubLoggingObj: + def _response_cost_calculator(self, result): + return 0.00042 + + chunk = ( + b"event: message_delta\n" + b'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' + b'"usage":{"input_tokens":14,"output_tokens":8,"cache_read_input_tokens":3202}}\n\n' + ) + + result = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection( + chunk, "claude-haiku-4-5", _StubLoggingObj() + ) + + assert result != chunk + data_line = next(ln for ln in result.decode("utf-8").splitlines() if ln.startswith("data:")) + payload = json.loads(data_line.split("data:", 1)[1].strip()) + assert payload["usage"]["cost"] == 0.00042 + assert payload["usage"]["cache_read_input_tokens"] == 3202 + # --------------------------------------------------------------------------- # SSE keepalive during the time-to-first-token (issue #34819) From 6f4844bfd9dd50901e3e4542f296602c9c8c9e56 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 19 Aug 2026 03:58:24 +0000 Subject: [PATCH 011/226] refactor(cost): trim streamed cost helper docstrings to the non-obvious bits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 24 ++++++---------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f5299b273b4..d533df8616d 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3440,8 +3440,7 @@ class ProxyBaseLLMRequestProcessing: Args: chunk: The streaming chunk (dict, str, bytes, or bytearray) model_name: Model name for cost calculation - litellm_logging_obj: The call's logging object, used to price the chunk with - the same custom/deployment pricing as the logging callback + litellm_logging_obj: The call's logging object, used for pricing Returns: The processed chunk with cost information injected if applicable @@ -3538,14 +3537,8 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _stream_usage_for_event(obj: Mapping[str, object], usage: Mapping[str, Any]) -> Usage | None: - """ - Build the ``Usage`` to price a streamed usage event. - - Anthropic goes through ``AnthropicConfig.calculate_usage``, the same transformation - the non-streaming path uses, so ``prompt_tokens`` is the full input total and the - cache read plus 5m/1h cache creation split land in ``prompt_tokens_details`` where - the pricer looks for them. - """ + # Anthropic reports input_tokens excluding cache tokens, so reuse the non-streaming + # transformation to total the prompt and keep the 5m/1h cache creation split if obj.get("type") == "message_delta": return AnthropicConfig().calculate_usage(usage_object=usage, reasoning_content=None) if obj.get("object") == "chat.completion.chunk": @@ -3580,12 +3573,8 @@ class ProxyBaseLLMRequestProcessing: service_tier: str | None, litellm_logging_obj: LiteLLMLoggingObj | None, ) -> float | None: - """ - Price a streamed usage response through the call's logging object when there is one, - so the streamed cost picks up the same custom/deployment pricing (``custom_pricing``, - ``custom_llm_provider``, ``base_model``, ``router_model_id``) as the logging callback - rather than the model's sticker price; fall back to ``completion_cost`` by model name. - """ + # Pricing via the logging object inherits the deployment's custom pricing, so the + # streamed cost matches what the logging callback records instead of sticker price cost_from_logging_obj: Final = ( ProxyBaseLLMRequestProcessing._logging_obj_cost_or_none(model_response, litellm_logging_obj) if litellm_logging_obj is not None @@ -3606,8 +3595,7 @@ class ProxyBaseLLMRequestProcessing: Args: obj: Dictionary containing the SSE event data model_name: Model name for cost calculation - litellm_logging_obj: The call's logging object, used so the injected cost - matches the cost the logging callback records + litellm_logging_obj: The call's logging object, used for pricing Returns: Modified dictionary with cost injected, or None if no modification needed From 087cdcff07ae34cd2d5ae9b2b4cd2282610d3953 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:18:12 +0000 Subject: [PATCH 012/226] feat: add bedrock grok 4.6 to model cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 54 +++++++++++++++++++ model_prices_and_context_window.json | 54 +++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 07f9027313b..b00ab87bf43 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -47379,6 +47379,60 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock_mantle/xai.grok-4.6": { + "use_openai_responses_path": true, + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 6.6e-06, + "cache_read_input_token_cost": 5.5e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + }, + "us.xai.grok-4.6": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 6.6e-06, + "cache_read_input_token_cost": 5.5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + }, + "global.xai.grok-4.6": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", "max_input_tokens": 256000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 07f9027313b..b00ab87bf43 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -47379,6 +47379,60 @@ "supports_vision": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock_mantle/xai.grok-4.6": { + "use_openai_responses_path": true, + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 6.6e-06, + "cache_read_input_token_cost": 5.5e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + }, + "us.xai.grok-4.6": { + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 6.6e-06, + "cache_read_input_token_cost": 5.5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + }, + "global.xai.grok-4.6": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", "max_input_tokens": 256000, From 1f6bef79cac5d33a39678e4e4df3bf11cba0b5b1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:24:29 +0000 Subject: [PATCH 013/226] fix: drop source url from grok 4.6 cost map entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 9 +++------ model_prices_and_context_window.json | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b00ab87bf43..872ef142123 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -47398,8 +47398,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + "supports_vision": true }, "us.xai.grok-4.6": { "input_cost_per_token": 2.2e-06, @@ -47414,8 +47413,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + "supports_vision": true }, "global.xai.grok-4.6": { "input_cost_per_token": 2e-06, @@ -47430,8 +47428,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + "supports_vision": true }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b00ab87bf43..872ef142123 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -47398,8 +47398,7 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + "supports_vision": true }, "us.xai.grok-4.6": { "input_cost_per_token": 2.2e-06, @@ -47414,8 +47413,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + "supports_vision": true }, "global.xai.grok-4.6": { "input_cost_per_token": 2e-06, @@ -47430,8 +47428,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-6.html" + "supports_vision": true }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", From aa832d81e91bb17e0f5ab081431cb03d2ef4083f Mon Sep 17 00:00:00 2001 From: ljogeiger <39981740+ljogeiger@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:08:42 +0000 Subject: [PATCH 014/226] fix(vertex_ai): only fall back to a placeholder thought signature on the first parallel function call Gemini returns a thoughtSignature on the first function call of a parallel batch and leaves the siblings bare. When replaying that assistant turn, litellm gave every unsigned call the skip_thought_signature_validator placeholder, so a three-call turn went back with three signatures where Gemini had produced one. Keep the placeholder for the first call only and forward the siblings with whatever signature they actually carry, which is usually none. --- .../prompt_templates/factory.py | 23 ++-- .../test_vertex_ai_gemini_transformation.py | 115 ++++++++++++++++++ 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 0ed15c43ccf..0a9d7c427b4 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1200,13 +1200,14 @@ def _encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: st return tool_call_id -def _get_thought_signature_from_tool(tool: dict, model: str | None = None) -> str | None: +def _get_thought_signature_from_tool(tool: dict) -> str | None: """Extract thought signature from tool call's provider_specific_fields. If not provided try to extract thought signature from tool call id Checks both tool.provider_specific_fields and tool.function.provider_specific_fields. - If no signature is found and model is gemini-3, returns a dummy signature. + Returns None when the tool call carries no signature; callers decide whether a + placeholder signature is needed. """ # First check tool's provider_specific_fields provider_fields: Final = tool.get("provider_specific_fields") or {} @@ -1236,13 +1237,6 @@ def _get_thought_signature_from_tool(tool: dict, model: str | None = None) -> st if len(parts) == 2: _, signature = parts return signature - # If no signature found and model is gemini-3, return dummy signature - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - - if model and VertexGeminiConfig._is_gemini_3_or_newer(model): - return _get_dummy_thought_signature() return None @@ -1312,8 +1306,10 @@ def convert_to_gemini_tool_call_invoke( VertexGeminiConfig, ) + needs_dummy_signature: Final = model is not None and VertexGeminiConfig._is_gemini_3_or_newer(model) + if tool_calls is not None: - for idx, tool in enumerate(tool_calls): + for tool in tool_calls: if "function" in tool: gemini_function_call: VertexFunctionCall | None = _gemini_tool_call_invoke_helper( function_call_params=tool["function"], @@ -1321,7 +1317,10 @@ def convert_to_gemini_tool_call_invoke( ) if gemini_function_call is not None: part_dict: VertexPartType = {"function_call": gemini_function_call} - thought_signature = _get_thought_signature_from_tool(dict(tool), model=model) + thought_signature = _get_thought_signature_from_tool(dict(tool)) + is_first_function_call = len(_parts_list) == 0 + if not thought_signature and is_first_function_call and needs_dummy_signature: + thought_signature = _get_dummy_thought_signature() if thought_signature: part_dict["thoughtSignature"] = thought_signature @@ -1344,7 +1343,7 @@ def convert_to_gemini_tool_call_invoke( thought_signature = provider_fields.get("thought_signature") # If no signature found and model is gemini-3, use dummy signature - if not thought_signature and model and VertexGeminiConfig._is_gemini_3_or_newer(model): + if not thought_signature and needs_dummy_signature: thought_signature = _get_dummy_thought_signature() if thought_signature: diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 8ee8186f6bb..28c551ab824 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1,3 +1,5 @@ +import base64 + from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_gemini_tool_call_result, ) @@ -784,6 +786,119 @@ def test_dummy_signature_with_function_call_mode(): assert gemini_parts[0]["thoughtSignature"] == expected_dummy +def _parallel_tool_calls(*signatures): + return [ + { + "id": f"call_{idx}", + "type": "function", + "function": { + "name": f"tool_{idx}", + "arguments": '{"location": "Paris"}', + **( + {"provider_specific_fields": {"thought_signature": signature}} + if signature is not None + else {} + ), + }, + "index": idx, + } + for idx, signature in enumerate(signatures) + ] + + +REAL_THOUGHT_SIGNATURE = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n" + + +def test_dummy_signature_only_on_first_parallel_tool_call(): + """Gemini only returns a thought signature on the first of N parallel function calls. + + The sibling calls carry no signature, so replaying them must not fabricate one. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, None, None), + }, + model="gemini-3-pro-preview", + ) + + expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode( + "utf-8" + ) + assert len(gemini_parts) == 3 + assert gemini_parts[0]["thoughtSignature"] == expected_dummy + assert "thoughtSignature" not in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[2] + + +def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty(): + """The real signature from Gemini rides on the first call; siblings stay signature-free.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(REAL_THOUGHT_SIGNATURE, None, None), + }, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 3 + assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[2] + + +def test_real_signature_on_later_parallel_tool_call_is_preserved(): + """A signature attached to a non-first call is still forwarded as-is.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, REAL_THOUGHT_SIGNATURE), + }, + model="gemini-3-pro-preview", + ) + + expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode( + "utf-8" + ) + assert len(gemini_parts) == 2 + assert gemini_parts[0]["thoughtSignature"] == expected_dummy + assert gemini_parts[1]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + + +def test_no_signatures_on_parallel_tool_calls_for_gemini_2_5(): + """Non-gemini-3 models never get a placeholder signature, on any call.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, None), + }, + model="gemini-2.5-flash", + ) + + assert len(gemini_parts) == 2 + assert all("thoughtSignature" not in part for part in gemini_parts) + + # Tests for media_resolution (detail parameter) handling - Issue #17084 class TestMediaResolution: """Tests for media_resolution handling in Gemini 2.x models""" From 677ef1e317080c20aec895a7ed25c058a1e18582 Mon Sep 17 00:00:00 2001 From: ljogeiger <39981740+ljogeiger@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:21:31 +0000 Subject: [PATCH 015/226] docs(vertex_ai): drop stale note about the removed model argument --- litellm/llms/vertex_ai/gemini/transformation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index f2d318a9ffd..11c026010ee 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -645,10 +645,9 @@ def _collect_tool_call_thought_signatures( the text part as well would send two copies and double-bill the previous turn's reasoning tokens on gemini-3 and newer models. - Detection deliberately calls _get_thought_signature_from_tool without the - model argument: with a gemini-3 model that helper synthesizes a dummy - signature for unsigned tool calls, which must not suppress a real - text-part signature (e.g. replaying gemini-2.5 history to a newer model). + Only real signatures count here; a synthesized placeholder must not + suppress a genuine text-part signature (e.g. replaying gemini-2.5 history + to a newer model). """ signatures: tuple[str, ...] = () From d5af42717e9713771b8a014455b79d22b0756754 Mon Sep 17 00:00:00 2001 From: ljogeiger <39981740+ljogeiger@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:32:24 +0000 Subject: [PATCH 016/226] test(vertex_ai): cover id-embedded, tool-level, and end-to-end parallel signature replay --- .../test_vertex_ai_gemini_transformation.py | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 28c551ab824..562f27c11cf 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -806,6 +806,27 @@ def _parallel_tool_calls(*signatures): ] +def _parallel_tool_calls_signed_via_id(*signatures): + """Parallel tool calls in the shape LiteLLM actually hands back to clients. + + The signature rides in the tool call id behind __thought__, which is what an + OpenAI-format client echoes back on the next turn. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + _encode_tool_call_id_with_signature, + ) + + return [ + { + "id": _encode_tool_call_id_with_signature(f"call_{idx}", signature), + "type": "function", + "function": {"name": f"tool_{idx}", "arguments": '{"location": "Paris"}'}, + "index": idx, + } + for idx, signature in enumerate(signatures) + ] + + REAL_THOUGHT_SIGNATURE = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n" @@ -899,6 +920,166 @@ def test_no_signatures_on_parallel_tool_calls_for_gemini_2_5(): assert all("thoughtSignature" not in part for part in gemini_parts) +def test_signature_embedded_in_tool_call_id_only_on_first_parallel_call(): + """The production shape: the signature arrives inside the first call's id, siblings have bare ids.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls_signed_via_id( + REAL_THOUGHT_SIGNATURE, None, None + ), + }, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 3 + assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[2] + + +def test_tool_level_provider_specific_fields_signature_leaves_siblings_empty(): + """A signature on the tool call itself, rather than on its function, behaves the same way.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + tool_calls = _parallel_tool_calls(None, None) + tool_calls[0]["provider_specific_fields"] = { + "thought_signature": REAL_THOUGHT_SIGNATURE + } + + gemini_parts = convert_to_gemini_tool_call_invoke( + {"role": "assistant", "content": None, "tool_calls": tool_calls}, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 2 + assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + + +def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry(): + """A non-function entry (e.g. an OpenAI custom tool call) emits no part, so it must not + consume the one placeholder slot and leave the real first function call bare.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _get_dummy_thought_signature, + convert_to_gemini_tool_call_invoke, + ) + + tool_calls = [ + {"id": "call_custom", "type": "custom", "custom": {"name": "noop", "input": ""}} + ] + _parallel_tool_calls(None, None) + + gemini_parts = convert_to_gemini_tool_call_invoke( + {"role": "assistant", "content": None, "tool_calls": tool_calls}, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 2 + assert gemini_parts[0]["thoughtSignature"] == _get_dummy_thought_signature() + assert "thoughtSignature" not in gemini_parts[1] + + +def test_no_placeholder_when_model_is_unknown(): + """Without a model there is nothing to prove the target needs a placeholder, so none is added.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, None), + }, + ) + + assert len(gemini_parts) == 2 + assert all("thoughtSignature" not in part for part in gemini_parts) + + +def test_real_signature_forwarded_to_gemini_2_5_without_placeholder_siblings(): + """Older models still receive a real signature that a client replays, and still get no placeholder.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(REAL_THOUGHT_SIGNATURE, None), + }, + model="gemini-2.5-flash", + ) + + assert len(gemini_parts) == 2 + assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + + +def test_parallel_tool_call_history_replayed_through_full_message_conversion(): + """End to end through the message-history converter, the path a real /chat/completions replay takes.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + messages = [ + {"role": "user", "content": "Weather in Paris, London and Tokyo?"}, + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls_signed_via_id( + REAL_THOUGHT_SIGNATURE, None, None + ), + }, + ] + + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-3-pro-preview" + ) + + model_parts = contents[1]["parts"] + assert len(model_parts) == 3 + assert model_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in model_parts[1] + assert "thoughtSignature" not in model_parts[2] + + +def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls(): + """gemini-2.5 history with a signed text part and parallel unsigned calls: the text keeps its real + signature, only the first call gets the placeholder, and the siblings stay bare.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _get_dummy_thought_signature, + ) + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + msg = { + "role": "assistant", + "content": "Checking all three cities.", + "provider_specific_fields": {"thought_signatures": ["real_25_signature"]}, + "tool_calls": _parallel_tool_calls(None, None, None), + } + + parts = _gemini_convert_messages_with_history( + messages=[msg], model="gemini-3-pro-preview" + )[0]["parts"] + + assert parts[0]["text"] == "Checking all three cities." + assert parts[0]["thoughtSignature"] == "real_25_signature" + assert parts[1]["thoughtSignature"] == _get_dummy_thought_signature() + assert "thoughtSignature" not in parts[2] + assert "thoughtSignature" not in parts[3] + + # Tests for media_resolution (detail parameter) handling - Issue #17084 class TestMediaResolution: """Tests for media_resolution handling in Gemini 2.x models""" From db50e123d5f23d475dab0bc62e33364ea19df817 Mon Sep 17 00:00:00 2001 From: ljogeiger <39981740+ljogeiger@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:42:24 +0000 Subject: [PATCH 017/226] test(vertex_ai): parametrize placeholder scoping across gemini-3 model variants --- .../test_vertex_ai_gemini_transformation.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 562f27c11cf..9ebf6db11de 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1,5 +1,7 @@ import base64 +import pytest + from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_gemini_tool_call_result, ) @@ -1052,6 +1054,40 @@ def test_parallel_tool_call_history_replayed_through_full_message_conversion(): assert "thoughtSignature" not in model_parts[2] +@pytest.mark.parametrize( + "model", + [ + "gemini-3-pro-preview", + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gemini-3.6-flash", + "gemini-3.7-flash", + "vertex_ai/gemini-3.7-flash", + "gemini/gemini-3.7-flash", + ], +) +def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model): + """Every gemini-3 family member, bare or provider-prefixed, gets one placeholder at most.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _get_dummy_thought_signature, + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, None, None), + }, + model=model, + ) + + assert len(gemini_parts) == 3 + assert gemini_parts[0]["thoughtSignature"] == _get_dummy_thought_signature() + assert "thoughtSignature" not in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[2] + + def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls(): """gemini-2.5 history with a signed text part and parallel unsigned calls: the text keeps its real signature, only the first call gets the placeholder, and the siblings stay bare.""" From 579291774b0a8b5e98c33140ae89fe60a30360b0 Mon Sep 17 00:00:00 2001 From: ljogeiger <39981740+ljogeiger@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:16:36 +0000 Subject: [PATCH 018/226] docs(vertex_ai): cite Google's thought signature rules for parallel calls Link the Gemini Enterprise Agent Platform docs at both places the behavior is decided. The docs state that only the first functionCall part of a parallel batch carries a thought_signature, and that setting skip_thought_signature_validator "should be a last resort as it will negatively impact model performance". --- .../litellm_core_utils/prompt_templates/factory.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 0a9d7c427b4..b676077ab0e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1245,10 +1245,14 @@ def _get_dummy_thought_signature() -> str: This is used when transferring conversation history from older models (like gemini-2.5-flash) to gemini-3, which requires thought_signature - for strict validation. + for strict validation. Google documents it as a last resort that "will + negatively impact model performance", so callers must only fall back to it + when no real signature is available. + + See: + https://ai.google.dev/gemini-api/docs/thought-signatures#faqs + https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures """ - # Return a base64-encoded dummy signature string - # Below dummy signature is recommended by google - https://ai.google.dev/gemini-api/docs/thought-signatures#faqs dummy_data: Final = b"skip_thought_signature_validator" return base64.b64encode(dummy_data).decode("utf-8") @@ -1318,6 +1322,9 @@ def convert_to_gemini_tool_call_invoke( if gemini_function_call is not None: part_dict: VertexPartType = {"function_call": gemini_function_call} thought_signature = _get_thought_signature_from_tool(dict(tool)) + # Gemini signs only the first functionCall part of a parallel batch, so scope the + # placeholder fallback to that part instead of fabricating one per sibling call: + # https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example is_first_function_call = len(_parts_list) == 0 if not thought_signature and is_first_function_call and needs_dummy_signature: thought_signature = _get_dummy_thought_signature() From a5ad22b8a3f734b09ce5e05a68dab5c5205907a4 Mon Sep 17 00:00:00 2001 From: ljogeiger <39981740+ljogeiger@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:16:36 +0000 Subject: [PATCH 019/226] test(vertex_ai): cover gemini-3.5-flash and drop assertion-echoing docstrings Add gemini-3.5-flash to the placeholder-scoping matrix and a regression test that a natively signed parallel turn replays with no skip_thought_signature_validator anywhere in the payload, the shape that was producing empty text responses on 3.5. Hoist the repeated placeholder expression into one constant and rewrite the docstrings that restated their own assertions to say why the case matters instead. --- .../test_vertex_ai_gemini_transformation.py | 83 +++++++++++++------ 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 9ebf6db11de..8c1de12e7d9 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -830,13 +830,14 @@ def _parallel_tool_calls_signed_via_id(*signatures): REAL_THOUGHT_SIGNATURE = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n" +PLACEHOLDER_SIGNATURE = base64.b64encode(b"skip_thought_signature_validator").decode( + "utf-8" +) def test_dummy_signature_only_on_first_parallel_tool_call(): - """Gemini only returns a thought signature on the first of N parallel function calls. - - The sibling calls carry no signature, so replaying them must not fabricate one. - """ + """Google documents the placeholder as a last resort that degrades quality, so an unsigned + parallel turn replayed to gemini-3 gets a budget of exactly one.""" from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_gemini_tool_call_invoke, ) @@ -850,17 +851,15 @@ def test_dummy_signature_only_on_first_parallel_tool_call(): model="gemini-3-pro-preview", ) - expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode( - "utf-8" - ) assert len(gemini_parts) == 3 - assert gemini_parts[0]["thoughtSignature"] == expected_dummy + assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE assert "thoughtSignature" not in gemini_parts[1] assert "thoughtSignature" not in gemini_parts[2] def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty(): - """The real signature from Gemini rides on the first call; siblings stay signature-free.""" + """Gemini signs only the first of N parallel function calls, so a faithful replay has + nothing to attach to the siblings.""" from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_gemini_tool_call_invoke, ) @@ -881,7 +880,8 @@ def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty(): def test_real_signature_on_later_parallel_tool_call_is_preserved(): - """A signature attached to a non-first call is still forwarded as-is.""" + """Clients may reorder or drop calls, so a signature that lands on a non-first call is + still the model's own and must survive the round trip.""" from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_gemini_tool_call_invoke, ) @@ -895,11 +895,8 @@ def test_real_signature_on_later_parallel_tool_call_is_preserved(): model="gemini-3-pro-preview", ) - expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode( - "utf-8" - ) assert len(gemini_parts) == 2 - assert gemini_parts[0]["thoughtSignature"] == expected_dummy + assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE assert gemini_parts[1]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE @@ -970,7 +967,6 @@ def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry(): """A non-function entry (e.g. an OpenAI custom tool call) emits no part, so it must not consume the one placeholder slot and leave the real first function call bare.""" from litellm.litellm_core_utils.prompt_templates.factory import ( - _get_dummy_thought_signature, convert_to_gemini_tool_call_invoke, ) @@ -984,7 +980,7 @@ def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry(): ) assert len(gemini_parts) == 2 - assert gemini_parts[0]["thoughtSignature"] == _get_dummy_thought_signature() + assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE assert "thoughtSignature" not in gemini_parts[1] @@ -1054,22 +1050,62 @@ def test_parallel_tool_call_history_replayed_through_full_message_conversion(): assert "thoughtSignature" not in model_parts[2] +@pytest.mark.parametrize( + "model", + ["gemini-3.5-flash", "vertex_ai/gemini-3.5-flash", "gemini/gemini-3.5-flash"], +) +def test_natively_signed_parallel_turn_never_carries_a_placeholder(model): + """A native gemini-3.5 parallel turn replays with zero skip_thought_signature_validator parts. + + Fabricating the placeholder alongside a real signature is what produced empty text responses + on gemini-3.5 parallel function calling, so the whole payload has to stay placeholder-free. + """ + import json + + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + messages = [ + {"role": "user", "content": "Weather in Paris, London and Tokyo?"}, + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls_signed_via_id( + REAL_THOUGHT_SIGNATURE, None, None + ), + }, + ] + + contents = _gemini_convert_messages_with_history(messages=messages, model=model) + + model_parts = contents[1]["parts"] + assert len(model_parts) == 3 + assert model_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in model_parts[1] + assert "thoughtSignature" not in model_parts[2] + assert PLACEHOLDER_SIGNATURE not in json.dumps(contents) + + @pytest.mark.parametrize( "model", [ "gemini-3-pro-preview", "gemini-3-flash-preview", "gemini-3.1-pro-preview", + "gemini-3.5-flash", "gemini-3.6-flash", "gemini-3.7-flash", + "vertex_ai/gemini-3.5-flash", "vertex_ai/gemini-3.7-flash", + "gemini/gemini-3.5-flash", "gemini/gemini-3.7-flash", ], ) def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model): - """Every gemini-3 family member, bare or provider-prefixed, gets one placeholder at most.""" + """The gemini-3 gate is a substring match, so every family member and prefix form has to + land on the same one-placeholder budget rather than only the versions we happened to try.""" from litellm.litellm_core_utils.prompt_templates.factory import ( - _get_dummy_thought_signature, convert_to_gemini_tool_call_invoke, ) @@ -1083,17 +1119,14 @@ def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model): ) assert len(gemini_parts) == 3 - assert gemini_parts[0]["thoughtSignature"] == _get_dummy_thought_signature() + assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE assert "thoughtSignature" not in gemini_parts[1] assert "thoughtSignature" not in gemini_parts[2] def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls(): - """gemini-2.5 history with a signed text part and parallel unsigned calls: the text keeps its real - signature, only the first call gets the placeholder, and the siblings stay bare.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - _get_dummy_thought_signature, - ) + """Text-part and function-call signatures are collected by separate code paths, so scoping the + placeholder must not disturb a real signature that arrived on the text part.""" from litellm.llms.vertex_ai.gemini.transformation import ( _gemini_convert_messages_with_history, ) @@ -1111,7 +1144,7 @@ def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls(): assert parts[0]["text"] == "Checking all three cities." assert parts[0]["thoughtSignature"] == "real_25_signature" - assert parts[1]["thoughtSignature"] == _get_dummy_thought_signature() + assert parts[1]["thoughtSignature"] == PLACEHOLDER_SIGNATURE assert "thoughtSignature" not in parts[2] assert "thoughtSignature" not in parts[3] From 367dd537b995d23421fd99bdb1093e443e98911e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:39:15 -0700 Subject: [PATCH 020/226] feat(e2e): move record/replay to the provider edge (LIT-5745) Replaces the test-side fixture transport with an in-process provider-edge HTTP server the proxy's deployments point their api_base at. Record forwards provider calls verbatim and writes them to the bundle; replay answers them from the bundle with zero provider calls while key auth, routing, cost calculation, and spend-log writes still execute against the live proxy and database. Drift comes back as HTTP 599 naming the computed and closest recorded keys. Request headers are never stored and responses are kept byte-identical between modes from the proxy's side of the socket. --- tests/e2e/CLAUDE.md | 12 +- tests/e2e/CONTRIBUTING.md | 10 +- tests/e2e/conftest.py | 14 +- tests/e2e/e2e_config.py | 32 +- tests/e2e/e2e_http.py | 34 + tests/e2e/fixture_bundle.py | 133 +--- tests/e2e/fixture_mode.py | 132 ++++ tests/e2e/fixture_transport.py | 724 ------------------ tests/e2e/provider_edge.py | 546 +++++++++++++ tests/e2e/proxy_client.py | 16 +- .../test_provider_edge_spend_e2e.py | 50 ++ tests/e2e/test_fixture_bundle.py | 46 +- tests/e2e/test_fixture_mode.py | 114 +++ tests/e2e/test_fixture_transport.py | 676 ---------------- tests/e2e/test_provider_edge.py | 492 ++++++++++++ 15 files changed, 1453 insertions(+), 1578 deletions(-) create mode 100644 tests/e2e/fixture_mode.py delete mode 100644 tests/e2e/fixture_transport.py create mode 100644 tests/e2e/provider_edge.py create mode 100644 tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py create mode 100644 tests/e2e/test_fixture_mode.py delete mode 100644 tests/e2e/test_fixture_transport.py create mode 100644 tests/e2e/test_provider_edge.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index d7334552d0c..840a40a54cd 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -73,13 +73,17 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover ## Record and replay fixtures -`E2E_FIXTURE_MODE` selects the transport every client is built on: `live` (the default, and what an unset variable means: nothing changes), `record` (run against the live proxy and write every interaction to a fixture bundle), or `replay` (serve every interaction back from the bundle with no HTTP at all, so a replay run needs no proxy and cannot bill a provider). The seam is `select_transport` in `fixture_transport.py`, applied inside `build_proxy_client`; both transports fulfil the same `Transport` protocol, so no test or client changes shape in any mode +`E2E_FIXTURE_MODE` scopes the proxy's provider-bound traffic: `live` (the default, and what an unset variable means: nothing changes), `record` (the proxy's provider calls are forwarded to the real provider through a local edge server and written to a fixture bundle), or `replay` (the edge answers those calls from the bundle, so the run makes zero provider calls and spends nothing). Test-to-proxy traffic always goes over the wire in every mode: record and replay both need the live proxy and database, because the point is that key auth, routing, cost calculation, and spend-log writes execute for real while only the provider is swapped out. Breaking any of those in the proxy turns a replay run red -A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per transport call in call order (`0000-post-chat-completions.json`). Auth header values and credential request fields (`api_key`, `*_secret_key`, `static_headers`, and the like; the list is `fixture_canonical.py`'s) are redacted on write, and file uploads store a sha256 digest instead of the bytes; response bodies are stored verbatim (a /key/generate response keeps the ephemeral virtual key it minted), which is part of why bundles are gitignored. `fixture_bundle.py` owns the format +The seam is `provider_edge.py`: `start_provider_edge` boots an in-process HTTP server (one shared instance per pytest process, `e2e_config.provider_edge_base` is the accessor) that mounts each supported provider under a path prefix (`EDGE_MOUNTS`: `/openai` -> `https://api.openai.com`, `/anthropic` -> `https://api.anthropic.com`). A test participates by registering its deployment with `api_base=provider_edge_base("openai")` plus the provider's path suffix; `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` is the reference. In live mode the accessor returns None and the deployment defaults to the real provider, so an edge-wired test runs in all three modes unchanged. Non-wired tests hit their providers live in every mode. The edge binds `E2E_PROVIDER_EDGE_BIND_HOST` (default 127.0.0.1) and advertises `E2E_PROVIDER_EDGE_ADVERTISE_HOST` in the api_base it hands out, for proxies running in containers -Replay matches calls per test by canonical key: `fixture_canonical.py` canonicalizes the recorded request (volatile headers and credential fields out, unique markers, generated ids, uuids, and timestamps replaced with fixed placeholders, object keys sorted) and the key is the method, path, and a content hash, so identity survives re-records and machine changes while any real content drift is a `ReplayMiss` that names the computed key, the closest recorded key with its file, and a content diff, and never falls through to a live call. Matching is order-independent across distinct keys (concurrent calls may interleave) and FIFO within one key (a poll loop replays its responses in recorded order); a passed test must also consume its whole recording, or teardown fails it naming a leftover key. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Every rewrite rule lives in `fixture_canonical.py`, so a new volatile header, credential field name, or generated-id shape is one edit there. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live proxy +A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. `fixture_bundle.py` owns the format. Record serves the proxy the same filtered stored response replay will serve later, so the two modes are byte-identical from the proxy's side of the socket -Deliberately not here yet: streaming chunk fidelity (LIT-5742) and scoping record/replay to provider-bound traffic (LIT-5745) +Replay matches calls per test by canonical key: `fixture_canonical.py` canonicalizes the recorded request (volatile headers and credential fields out, unique markers, generated ids, uuids, and timestamps replaced with fixed placeholders, object keys sorted) and the key is the method, edge path, and a content hash, so identity survives re-records and machine changes while any real content drift comes back as an HTTP 599 naming the computed key, the closest recorded key with its file, and a content diff, and never falls through to a live call. Matching is order-independent across distinct keys (concurrent calls may interleave) and FIFO within one key (a retry loop replays its responses in recorded order); a passed test must also consume its whole recording, or teardown fails it naming a leftover key. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Every rewrite rule lives in `fixture_canonical.py`, so a new volatile header, credential field name, or generated-id shape is one edit there. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live providers + +A replayed response carries the recorded provider response id, and `LiteLLM_SpendLogs.request_id` (the table's primary key) is that id, so a replay against a database that still holds the record run's rows silently dedupes its spend inserts and any spend assertion goes red with zero matching rows and nothing in the proxy log. Run both modes with `E2E_RESET_SPEND_LOGS=1` (plus `DATABASE_URL` in the runner env) so each session truncates the table after itself, or replay against a fresh database, which is the CI shape + +Current limits: streaming chunk fidelity is LIT-5742 (a streamed response records as one buffered body), CI wiring is LIT-5748, Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), multipart uploads have per-run random boundaries (the digest changes every run, so they always miss), and deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base) ## Typing diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 67da1be9562..9096050a45a 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -54,14 +54,16 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT ### Record and replay -`E2E_FIXTURE_MODE=record` runs a suite against the live proxy as usual while writing every request/response pair to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`); `E2E_FIXTURE_MODE=replay` then runs the same suite entirely from that bundle, with no proxy traffic and no provider spend; the proxy liveness gate is skipped, so replay runs with no proxy up at all. Unset (or `live`) behaves exactly as before the knob existed +Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop ```bash -E2E_FIXTURE_MODE=record uv run pytest tests/e2e/llm_translation/ -v -E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/llm_translation/ -v +E2E_FIXTURE_MODE=record uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v +E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v ``` -Replay fails hard (`ReplayMiss`) when the tests drift from the recording, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. See `CLAUDE.md` in this directory for the bundle format and the transport seam +One sharp edge: a replayed response reuses the recorded provider response id, and that id is the primary key of `LiteLLM_SpendLogs`, so replaying against a database that still holds the record run's rows silently dedupes the spend writes and a spend assertion fails with zero rows. Run both commands above with `E2E_RESET_SPEND_LOGS=1` (and `DATABASE_URL` set in the pytest env) so each session truncates the spend log table after itself, or point replay at a fresh database + +Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the current limits (streaming, Bedrock, multipart) Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index da2a7da0bfa..dbe2d6e514e 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -23,12 +23,8 @@ import requests from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup -from fixture_transport import ( - fixture_mode_collection_error, - fixture_report_lines, - parse_fixture_mode, - replay_leftover_error, -) +from fixture_mode import fixture_mode_collection_error, fixture_report_lines +from provider_edge import replay_leftover_error from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from proxy_client import ProxyClient, build_proxy_client @@ -114,12 +110,10 @@ def _proxy_fail_reason() -> str | None: def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. Unmarked tests (unit coverage of the harness) don't touch the proxy, so they - run even when none is up. Never skip for a missing proxy. Replay mode serves - every call from the fixture bundle, so it needs no live proxy either.""" + run even when none is up. Never skip for a missing proxy. Replay mode needs + the proxy too: only provider-bound traffic replays from the bundle.""" if item.get_closest_marker("e2e") is None: return - if parse_fixture_mode(FIXTURE_MODE_RAW) == "replay": - return reason = _proxy_fail_reason() if reason is not None: pytest.fail(reason) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index a5c3729f4be..8bf39f6021f 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -13,7 +13,8 @@ from pathlib import Path from dotenv import load_dotenv -from fixture_transport import deterministic_marker, parse_fixture_mode +from fixture_mode import deterministic_marker, parse_fixture_mode +from provider_edge import provider_edge_api_base # Local runs keep provider / DataDog keys in tests/e2e/.env (see CONTRIBUTING.md). # Compose injects them into the proxy container, but pytest on the host does not @@ -92,15 +93,24 @@ PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15")) EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes") -# Record/replay fixture selection (see fixture_transport.py). The raw mode value -# is parsed and validated there; "live" (the default, also for empty values) -# means the harness behaves exactly as before this knob existed. +# Record/replay fixture selection (see fixture_mode.py and provider_edge.py). +# The raw mode value is parsed and validated there; "live" (the default, also +# for empty values) means the harness behaves exactly as before this knob +# existed. FIXTURE_MODE_RAW = os.environ.get("E2E_FIXTURE_MODE", "live") FIXTURE_DIR = Path( os.environ.get("E2E_FIXTURE_DIR", "").strip() or str(Path(__file__).resolve().parent / ".fixtures") ) +# Where the provider-edge server binds, and the host name edge api_base URLs +# advertise to the proxy. They differ when the proxy runs in a container and +# reaches the pytest host via a gateway name like host.docker.internal. +PROVIDER_EDGE_BIND_HOST = os.environ.get("E2E_PROVIDER_EDGE_BIND_HOST", "").strip() or "127.0.0.1" +PROVIDER_EDGE_ADVERTISE_HOST = ( + os.environ.get("E2E_PROVIDER_EDGE_ADVERTISE_HOST", "").strip() or PROVIDER_EDGE_BIND_HOST +) + # Deliberately modest concurrency. The suite shares its proxy with every other # suite in the run, and 750 users at spawn rate 50 saturated the request path hard # enough to distort latency-sensitive neighbours (and to spend real provider money @@ -157,6 +167,20 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str: return f"{base}?toolsets={toolsets}" if toolsets else base +def provider_edge_base(mount: str) -> str | None: + """The api_base an edge-wired deployment should register with, using this + process's fixture-mode and edge-host configuration: None in live mode, the + shared edge server's mount URL in record and replay.""" + return provider_edge_api_base( + mount, + mode_raw=FIXTURE_MODE_RAW, + bundle_dir=FIXTURE_DIR, + bind_host=PROVIDER_EDGE_BIND_HOST, + advertise_host=PROVIDER_EDGE_ADVERTISE_HOST, + forward_timeout=REQUEST_TIMEOUT, + ) + + def unique_marker() -> str: """A short unique token per call/run, so concurrent runs and the shared response cache never collide on prompts, tags, or customer ids. In record diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index cb6fc7a01e5..03f201e946e 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -647,3 +647,37 @@ def download( content_type=_hdr(resp, "content-type"), body=resp.text, ) + + +class RawResponse(BaseModel): + """A verbatim upstream HTTP response for the provider edge (provider_edge.py): + status, lowercased headers, raw bytes. No Result classification because the + edge relays provider errors to the proxy untouched.""" + + status_code: int + headers: dict[str, str] + body: bytes + + +def forward( + method: str, + url: str, + *, + headers: dict[str, str], + body: bytes | None, + timeout: float = 60.0, +) -> RawResponse | NetworkError: + """Relay one provider-bound request verbatim for the provider edge's record + mode. No retries, no redirects, no schema: the proxy owns retry policy and + the recorded bundle must hold exactly what the provider returned.""" + try: + resp = requests.request( + method, url, headers=headers, data=body, timeout=timeout, allow_redirects=False + ) + except requests.RequestException as exc: + return NetworkError(message=str(exc)) + return RawResponse( + status_code=resp.status_code, + headers={name.lower(): value for name, value in resp.headers.items()}, + body=resp.content, + ) diff --git a/tests/e2e/fixture_bundle.py b/tests/e2e/fixture_bundle.py index 615ae8df1a4..6feb40fc8bc 100644 --- a/tests/e2e/fixture_bundle.py +++ b/tests/e2e/fixture_bundle.py @@ -1,17 +1,18 @@ -"""On-disk fixture bundle format for record/replay e2e runs (LIT-5729). +"""On-disk fixture bundle format for record/replay e2e runs (LIT-5729/LIT-5745). A bundle is a directory: one ``manifest.json`` (record timestamp + harness version + format version) plus one subdirectory per test, holding one JSON file -per transport interaction in call order. Bundles older than +per provider-bound interaction in call order. Bundles older than ``MAX_BUNDLE_AGE`` hard-fail replay at collection time (see conftest), so a green replay run can never certify against fixtures that have drifted more than -a week from the live proxy. +a week from the live providers. -This module owns the format only. The transports that produce and consume it -live in fixture_transport.py and the canonical match keys they compute live in -fixture_canonical.py (LIT-5741); streaming chunk fidelity and provider-scoping -are follow-ups (LIT-5742/5745). Every interaction file stores the full redacted -request because replay matches on its canonicalized content. +This module owns the format only. The provider-edge server that produces and +consumes it lives in provider_edge.py (LIT-5745) and the canonical match keys +it computes live in fixture_canonical.py (LIT-5741); streaming chunk fidelity +is a follow-up (LIT-5742). Every interaction file stores the full redacted +request because replay matches on its canonicalized content, and the response +as the raw HTTP status, filtered headers, and base64 body the provider sent. """ from __future__ import annotations @@ -23,29 +24,14 @@ import subprocess from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Annotated, Final, Literal +from typing import Final -from pydantic import BaseModel, Field, JsonValue, TypeAdapter +from pydantic import BaseModel, JsonValue -from e2e_http import ( - BinaryStream, - NetworkError, - ProbeResult, - RateLimitedError, - Result, - StreamingResponse, - Success, - UnauthorizedError, - UnknownApiError, - ValidationError, -) - -BUNDLE_FORMAT_VERSION: Final = 1 +BUNDLE_FORMAT_VERSION: Final = 2 MAX_BUNDLE_AGE: Final = timedelta(days=7) MANIFEST_FILENAME: Final = "manifest.json" -_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) - class Manifest(BaseModel): format_version: int @@ -54,13 +40,14 @@ class Manifest(BaseModel): class RecordedRequest(BaseModel): - """The request as the transport saw it, auth header values and credential - body/form fields redacted. + """The provider-bound request as the edge saw it, headers empty (SDK + telemetry headers vary run to run and auth material never touches disk). Replay matches on the canonical content key fixture_canonical.py computes - over ``method`` (the transport verb, not the HTTP verb), ``path``, and the - canonicalized headers, params, body, form, and file identity. File uploads - store a content digest instead of the bytes.""" + over ``method``, ``path`` (the edge path including the provider mount, + query string excluded), and the canonicalized headers, params, body, form, + and file identity. Non-JSON bodies store a canonicalized content digest + instead of the bytes.""" method: str path: str @@ -73,85 +60,19 @@ class RecordedRequest(BaseModel): file_bytes: int | None = None -class RecordedResult(BaseModel): - """A ``Result[R]`` flattened for disk. ``data`` holds the success payload as - raw JSON; replay re-validates it against the ``response_type`` the caller - passes, exactly like a live response body.""" +class RecordedHttpResponse(BaseModel): + """The provider's raw HTTP response: status, headers minus hop-by-hop and + volatile entries (see provider_edge.py), and the body as base64 so binary + payloads survive JSON.""" - shape: Literal["result"] = "result" - kind: Literal["success", "network", "unauthorized", "rate_limited", "validation", "unknown"] - status_code: int | None = None - data: JsonValue | None = None - message: str | None = None - body: str | None = None - retry_after_seconds: int | None = None - - -class RecordedStreaming(BaseModel): - shape: Literal["streaming"] = "streaming" - payload: StreamingResponse - - -class RecordedBinary(BaseModel): - shape: Literal["binary"] = "binary" - payload: BinaryStream - - -class RecordedProbe(BaseModel): - shape: Literal["probe"] = "probe" - payload: ProbeResult - - -type RecordedResponse = RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe + status_code: int + headers: dict[str, str] + body_b64: str class Interaction(BaseModel): request: RecordedRequest - response: Annotated[ - RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe, - Field(discriminator="shape"), - ] - - -def to_json_value(model: BaseModel) -> JsonValue: - return _JSON.validate_json(model.model_dump_json(by_alias=True)) - - -def from_result[R: BaseModel](result: Result[R]) -> RecordedResult: - match result: - case Success(status_code=status_code, data=data): - return RecordedResult(kind="success", status_code=status_code, data=to_json_value(data)) - case NetworkError(message=message): - return RecordedResult(kind="network", message=message) - case UnauthorizedError(): - return RecordedResult(kind="unauthorized") - case RateLimitedError(retry_after_seconds=retry_after_seconds, body=body): - return RecordedResult(kind="rate_limited", retry_after_seconds=retry_after_seconds, body=body) - case ValidationError(message=message): - return RecordedResult(kind="validation", message=message) - case UnknownApiError(status_code=status_code, body=body): - return RecordedResult(kind="unknown", status_code=status_code, body=body) - - -def to_result[R: BaseModel](recorded: RecordedResult, response_type: type[R]) -> Result[R]: - match recorded.kind: - case "success": - return Success( - status_code=recorded.status_code or 200, - data=response_type.model_validate(recorded.data), - ) - case "network": - return NetworkError(message=recorded.message or "") - case "unauthorized": - return UnauthorizedError() - case "rate_limited": - return RateLimitedError( - retry_after_seconds=recorded.retry_after_seconds, body=recorded.body or "" - ) - case "validation": - return ValidationError(message=recorded.message or "") - case "unknown": - return UnknownApiError(status_code=recorded.status_code or 0, body=recorded.body or "") + response: RecordedHttpResponse def slugify(raw: str, *, limit: int = 60) -> str: @@ -198,7 +119,7 @@ class BundleRecorder: root: Path _ordinals: dict[str, int] = field(default_factory=dict) - def record(self, *, test_key: str, request: RecordedRequest, response: RecordedResponse) -> None: + def record(self, *, test_key: str, request: RecordedRequest, response: RecordedHttpResponse) -> None: slug = slug_for_test(test_key) ordinal = self._ordinals.get(slug, 0) self._ordinals[slug] = ordinal + 1 diff --git a/tests/e2e/fixture_mode.py b/tests/e2e/fixture_mode.py new file mode 100644 index 00000000000..110f44380b4 --- /dev/null +++ b/tests/e2e/fixture_mode.py @@ -0,0 +1,132 @@ +"""Fixture-mode selection and per-test determinism for record/replay e2e runs. + +``E2E_FIXTURE_MODE`` is live (the default; nothing changes), record, or replay. +This module owns everything mode-shaped that is independent of the provider +edge itself: parsing the raw env value, the collection-time gate that aborts a +run whose mode can never work (unknown value, or replay against a missing or +stale bundle), the pytest report-header lines, the running test's node id, and +the deterministic per-test marker that lets a replay run regenerate exactly +the requests the record run sent. The provider-edge server that records and +serves provider traffic lives in provider_edge.py (LIT-5745). +""" + +from __future__ import annotations + +import hashlib +import os +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Final, Literal, assert_never + +from fixture_bundle import ( + FreshBundle, + StaleBundle, + UnreadableBundle, + check_freshness, + format_age, +) + +type FixtureMode = Literal["live", "record", "replay"] + +FIXTURE_MODES: Final[tuple[FixtureMode, ...]] = ("live", "record", "replay") + +SESSION_TEST_KEY: Final = "session" + + +@dataclass(frozen=True, slots=True) +class InvalidFixtureMode: + value: str + + +def parse_fixture_mode(raw: str) -> FixtureMode | InvalidFixtureMode: + normalized = raw.strip().lower() or "live" + match normalized: + case "live" | "record" | "replay": + return normalized + case _: + return InvalidFixtureMode(value=raw) + + +def current_test_key() -> str: + """The pytest node id of the running test, from the PYTEST_CURRENT_TEST env + var pytest maintains (`` (setup|call|teardown)``); ``session`` for + calls outside any test (e.g. session-finish cleanup).""" + raw = os.environ.get("PYTEST_CURRENT_TEST", "") + if not raw: + return SESSION_TEST_KEY + return raw.rsplit(" (", 1)[0] + + +class ReplayMiss(AssertionError): + """Replay had no recorded interaction for a provider call the proxy made. + The suite drifted from the bundle (or the bundle from the suite): re-record.""" + + +_marker_ordinals: Final[dict[str, int]] = {} + + +def deterministic_marker() -> str: + """Stable stand-in for uuid-based unique markers in record and replay modes: + the Nth marker of a test is a pure function of the test's node id and N, so a + replay run regenerates exactly the model names, prompts, and tags the record + run sent and every recorded provider interaction still matches its key.""" + test_key = current_test_key() + ordinal = _marker_ordinals.get(test_key, 0) + _marker_ordinals[test_key] = ordinal + 1 + return hashlib.sha1(f"{test_key}#{ordinal}".encode()).hexdigest()[:12] + + +def fixture_mode_collection_error(mode_raw: str, bundle_dir: Path, *, now: datetime) -> str | None: + """Session-abort reason for a fixture-mode setup that can never work, or None. + Called at collection time (conftest pytest_sessionstart) so a stale or missing + bundle fails the whole run up front, naming the bundle age, instead of failing + every test individually.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode(value=value): + return f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}" + case "live" | "record": + return None + case "replay": + freshness = check_freshness(bundle_dir, now=now) + match freshness: + case FreshBundle(): + return None + case StaleBundle(recorded_at=recorded_at, age=age, limit=limit): + return ( + f"fixture bundle at {bundle_dir} is stale: recorded {recorded_at.isoformat()}, " + f"age {format_age(age)} exceeds the {limit.days}-day limit; " + "re-record with E2E_FIXTURE_MODE=record" + ) + case UnreadableBundle(reason=reason): + return f"E2E_FIXTURE_MODE=replay cannot use bundle at {bundle_dir}: {reason}" + case _: + assert_never(freshness) + case _: + assert_never(mode) + + +def fixture_report_lines(mode_raw: str, bundle_dir: Path, *, now: datetime) -> list[str]: + """pytest report-header lines; empty in live mode so an unset + E2E_FIXTURE_MODE keeps today's output byte-identical.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode() | "live": + return [] + case "record": + return [f"e2e fixture mode: record -> {bundle_dir}"] + case "replay": + freshness = check_freshness(bundle_dir, now=now) + match freshness: + case FreshBundle(manifest=manifest): + return [ + f"e2e fixture mode: replay <- {bundle_dir} " + f"(recorded {manifest.recorded_at.isoformat()}, harness {manifest.harness_version})" + ] + case StaleBundle() | UnreadableBundle(): + return [f"e2e fixture mode: replay <- {bundle_dir}"] + case _: + assert_never(freshness) + case _: + assert_never(mode) diff --git a/tests/e2e/fixture_transport.py b/tests/e2e/fixture_transport.py deleted file mode 100644 index ce4eec701ca..00000000000 --- a/tests/e2e/fixture_transport.py +++ /dev/null @@ -1,724 +0,0 @@ -"""Record/replay transports behind the same ``Transport`` protocol (LIT-5729). - -``RecordingTransport`` decorates the live transport: every call passes through -unchanged and its request/response pair is appended to the fixture bundle. -``ReplayTransport`` implements the protocol from a recorded bundle alone: no -HTTP, no proxy, no provider spend. Because both fulfil ``Transport``, no test -or client changes shape; ``build_proxy_client`` picks the transport from -``E2E_FIXTURE_MODE`` (live | record | replay, default live). - -Replay matches each call by test node id and canonical content key -(fixture_canonical.py, LIT-5741): volatile headers, credential fields, unique -markers, generated ids, and timestamps are canonicalized out before hashing, so -matching is order-independent across distinct keys, FIFO within a key, and a -miss fails hard (``ReplayMiss``) printing the computed key and the closest -recorded key without ever falling through to a live call. Streaming chunk -fidelity is LIT-5742; scoping record/replay to provider-bound traffic is -LIT-5745. -""" - -from __future__ import annotations - -import difflib -import functools -import hashlib -import os -from collections import deque -from dataclasses import dataclass, field -from datetime import datetime -from itertools import islice -from pathlib import Path -from typing import Final, Literal, assert_never - -from pydantic import BaseModel, JsonValue - -from e2e_http import AuthHeaders, BinaryStream, ProbeResult, Result, StreamingResponse -from fixture_bundle import ( - BundleRecorder, - FreshBundle, - Interaction, - LoadedBundle, - RecordedBinary, - RecordedProbe, - RecordedRequest, - RecordedResponse, - RecordedResult, - RecordedStreaming, - StaleBundle, - UnreadableBundle, - UnsafeBundleDir, - check_freshness, - format_age, - from_result, - interaction_filename, - load_bundle, - prepare_bundle, - slug_for_test, - to_json_value, - to_result, -) -from fixture_canonical import CanonicalRequest, canonicalize, is_secret_field -from transport import Transport - -type FixtureMode = Literal["live", "record", "replay"] - -FIXTURE_MODES: Final[tuple[FixtureMode, ...]] = ("live", "record", "replay") - -SESSION_TEST_KEY: Final = "session" - -REDACTED_HEADER_NAMES: Final[frozenset[str]] = frozenset({"authorization", "x-litellm-api-key"}) -REDACTED_VALUE: Final = "" - - -@dataclass(frozen=True, slots=True) -class InvalidFixtureMode: - value: str - - -def parse_fixture_mode(raw: str) -> FixtureMode | InvalidFixtureMode: - normalized = raw.strip().lower() or "live" - match normalized: - case "live" | "record" | "replay": - return normalized - case _: - return InvalidFixtureMode(value=raw) - - -def current_test_key() -> str: - """The pytest node id of the running test, from the PYTEST_CURRENT_TEST env - var pytest maintains (`` (setup|call|teardown)``); ``session`` for - calls outside any test (e.g. session-finish cleanup).""" - raw = os.environ.get("PYTEST_CURRENT_TEST", "") - if not raw: - return SESSION_TEST_KEY - return raw.rsplit(" (", 1)[0] - - -class ReplayMiss(AssertionError): - """Replay had no recorded interaction for a call the suite made. The test - drifted from the bundle (or the bundle from the suite): re-record.""" - - -_marker_ordinals: Final[dict[str, int]] = {} - - -def deterministic_marker() -> str: - """Stable stand-in for uuid-based unique markers in record and replay modes: - the Nth marker of a test is a pure function of the test's node id and N, so a - replay run regenerates exactly the model names, prompts, and tags the record - run sent and every recorded poll response still satisfies its predicate.""" - test_key = current_test_key() - ordinal = _marker_ordinals.get(test_key, 0) - _marker_ordinals[test_key] = ordinal + 1 - return hashlib.sha1(f"{test_key}#{ordinal}".encode()).hexdigest()[:12] - - -def _dump_flat(model: BaseModel | None) -> dict[str, str]: - if model is None: - return {} - dumped: dict[str, object] = model.model_dump(by_alias=True, exclude_none=True) - return {key: str(value) for key, value in dumped.items()} - - -def _redact(headers: dict[str, str]) -> dict[str, str]: - return { - name: REDACTED_VALUE if name.lower() in REDACTED_HEADER_NAMES else value - for name, value in headers.items() - } - - -def _redact_secret_fields(value: JsonValue) -> JsonValue: - match value: - case dict(): - return { - key: REDACTED_VALUE - if is_secret_field(key) and item is not None - else _redact_secret_fields(item) - for key, item in value.items() - } - case list(): - return [_redact_secret_fields(item) for item in value] - case _: - return value - - -def _redact_flat(fields: dict[str, str]) -> dict[str, str]: - return { - key: REDACTED_VALUE if is_secret_field(key) else value for key, value in fields.items() - } - - -def recorded_request( - method: str, - path: str, - *, - headers: BaseModel, - body: BaseModel | None = None, - params: BaseModel | None = None, - form: BaseModel | None = None, - file_name: str | None = None, - file_content: bytes | None = None, -) -> RecordedRequest: - return RecordedRequest( - method=method, - path=path, - headers=_redact(_dump_flat(headers)), - params=_redact_flat(_dump_flat(params)), - body=None if body is None else _redact_secret_fields(to_json_value(body)), - form=None if form is None else _redact_flat(_dump_flat(form)), - file_name=file_name, - file_sha256=None if file_content is None else hashlib.sha256(file_content).hexdigest(), - file_bytes=None if file_content is None else len(file_content), - ) - - -@dataclass(frozen=True, slots=True) -class RecordingTransport: - """Decorator over the live transport: forwards every call and appends the - interaction to the bundle, so a green live run leaves behind exactly the - traffic replay needs.""" - - inner: Transport - recorder: BundleRecorder - - def _record(self, request: RecordedRequest, response: RecordedResponse) -> None: - self.recorder.record(test_key=current_test_key(), request=request, response=response) - - def bearer(self, key: str) -> AuthHeaders: - return self.inner.bearer(key) - - @property - def master(self) -> AuthHeaders: - return self.inner.master - - def post[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - result = self.inner.post(path, headers=headers, json=json, response_type=response_type) - self._record(recorded_request("post", path, headers=headers, body=json), from_result(result)) - return result - - def get[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - params: BaseModel, - response_type: type[R], - timeout: float | None = None, - ) -> Result[R]: - result = self.inner.get( - path, headers=headers, params=params, response_type=response_type, timeout=timeout - ) - self._record(recorded_request("get", path, headers=headers, params=params), from_result(result)) - return result - - def delete[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - response_type: type[R], - params: BaseModel | None = None, - ) -> Result[R]: - result = self.inner.delete( - path, headers=headers, json=json, response_type=response_type, params=params - ) - self._record( - recorded_request("delete", path, headers=headers, body=json, params=params), - from_result(result), - ) - return result - - def patch[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - result = self.inner.patch(path, headers=headers, json=json, response_type=response_type) - self._record(recorded_request("patch", path, headers=headers, body=json), from_result(result)) - return result - - def put[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - result = self.inner.put(path, headers=headers, json=json, response_type=response_type) - self._record(recorded_request("put", path, headers=headers, body=json), from_result(result)) - return result - - def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: - response = self.inner.stream(path, headers=headers, json=json) - self._record( - recorded_request("stream", path, headers=headers, body=json), - RecordedStreaming(payload=response), - ) - return response - - def stream_binary( - self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 - ) -> BinaryStream: - response = self.inner.stream_binary(path, headers=headers, json=json, chunk_size=chunk_size) - self._record( - recorded_request("stream_binary", path, headers=headers, body=json), - RecordedBinary(payload=response), - ) - return response - - def send( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - params: BaseModel | None = None, - stream: bool = False, - ) -> StreamingResponse: - response = self.inner.send(path, headers=headers, json=json, params=params, stream=stream) - self._record( - recorded_request("send", path, headers=headers, body=json, params=params), - RecordedStreaming(payload=response), - ) - return response - - def probe(self, path: str, *, params: BaseModel) -> ProbeResult: - response = self.inner.probe(path, params=params) - self._record( - recorded_request("probe", path, headers=self.master, params=params), - RecordedProbe(payload=response), - ) - return response - - def upload[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - form: BaseModel, - filename: str, - content: bytes, - file_content_type: str = "application/jsonl", - file_field: str = "file", - params: BaseModel | None = None, - response_type: type[R], - ) -> Result[R]: - result = self.inner.upload( - path, - headers=headers, - form=form, - filename=filename, - content=content, - file_content_type=file_content_type, - file_field=file_field, - params=params, - response_type=response_type, - ) - self._record( - recorded_request( - "upload", - path, - headers=headers, - params=params, - form=form, - file_name=filename, - file_content=content, - ), - from_result(result), - ) - return result - - def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: - response = self.inner.download(path, headers=headers) - self._record( - recorded_request("download", path, headers=headers), - RecordedStreaming(payload=response), - ) - return response - - -def _build_pool(recorded: tuple[Interaction, ...]) -> dict[str, deque[Interaction]]: - keys: Final = tuple(canonicalize(interaction.request).key for interaction in recorded) - return { - key: deque( - interaction - for candidate_key, interaction in zip(keys, recorded, strict=True) - if candidate_key == key - ) - for key in dict.fromkeys(keys) - } - - -def _closest_recorded( - canonical: CanonicalRequest, recorded: tuple[Interaction, ...] -) -> tuple[CanonicalRequest, str]: - candidates: Final = tuple(canonicalize(interaction.request) for interaction in recorded) - ratios: Final = tuple( - difflib.SequenceMatcher( - None, f"{canonical.method} {canonical.path}\n{canonical.content}", - f"{candidate.method} {candidate.path}\n{candidate.content}", - ).ratio() - for candidate in candidates - ) - best: Final = max(range(len(candidates)), key=lambda index: ratios[index]) - return candidates[best], interaction_filename(best, recorded[best].request) - - -def _miss_message(test_key: str, slug: str, canonical: CanonicalRequest, bundle: LoadedBundle) -> str: - recorded: Final = bundle.interactions.get(slug, ()) - if not recorded: - return ( - f"replay miss for {test_key}: computed key {canonical.key} but nothing is recorded " - f"under {slug}; re-record with E2E_FIXTURE_MODE=record" - ) - closest, closest_file = _closest_recorded(canonical, recorded) - diff: Final = "\n".join( - islice( - difflib.unified_diff( - closest.pretty_content().splitlines(), - canonical.pretty_content().splitlines(), - fromfile=f"closest recorded ({closest_file})", - tofile="test made", - lineterm="", - ), - 60, - ) - ) - return ( - f"replay miss for {test_key}: no recorded interaction matches key {canonical.key}; " - f"closest recorded key is {closest.key} ({closest_file})\n{diff}\n" - "re-record with E2E_FIXTURE_MODE=record" - ) - - -@dataclass(slots=True) -class ReplaySource: - """One shared pool per test over a loaded bundle, so every client built in - the session consumes the same recorded interactions. Every pool is built - once at construction and per-key consumption is a single atomic deque pop, - so concurrent replay calls never race. Calls match by canonical content - key: order-independent across distinct keys (concurrent tests interleave - calls nondeterministically), FIFO within one key (a poll loop replays its - recorded responses in recorded order).""" - - bundle: LoadedBundle - _pools: dict[str, dict[str, deque[Interaction]]] = field(init=False) - - def __post_init__(self) -> None: - self._pools = { - slug: _build_pool(recorded) for slug, recorded in self.bundle.interactions.items() - } - - def _pool(self, slug: str) -> dict[str, deque[Interaction]]: - return self._pools.get(slug, {}) - - def next_interaction(self, request: RecordedRequest) -> Interaction: - test_key: Final = current_test_key() - slug: Final = slug_for_test(test_key) - pool: Final = self._pool(slug) - canonical: Final = canonicalize(request) - queue: Final = pool.get(canonical.key) - if queue is None: - raise ReplayMiss(_miss_message(test_key, slug, canonical, self.bundle)) - try: - return queue.popleft() - except IndexError: - raise ReplayMiss( - f"replay exhausted for {test_key}: every recorded interaction for key " - f"{canonical.key} is already consumed; re-record with E2E_FIXTURE_MODE=record" - ) from None - - def leftover_error(self, test_key: str) -> str | None: - """Non-None when the test consumed fewer interactions than were recorded, - meaning a passing replay proved less than the bundle claims.""" - slug: Final = slug_for_test(test_key) - recorded: Final = self.bundle.interactions.get(slug, ()) - if not recorded: - return None - leftover: Final = tuple( - interaction for queue in self._pool(slug).values() for interaction in queue - ) - if not leftover: - return None - return ( - f"replay incomplete for {test_key}: {len(leftover)} of {len(recorded)} recorded " - f"interactions never consumed, e.g. {canonicalize(leftover[0].request).key}; " - "re-record with E2E_FIXTURE_MODE=record" - ) - - -def _expect_result(interaction: Interaction) -> RecordedResult: - match interaction.response: - case RecordedResult() as recorded: - return recorded - case RecordedStreaming() | RecordedBinary() | RecordedProbe(): - raise ReplayMiss( - f"recorded {interaction.request.method} {interaction.request.path} is not a typed result" - ) - - -def _expect_streaming(interaction: Interaction) -> StreamingResponse: - match interaction.response: - case RecordedStreaming(payload=payload): - return payload - case RecordedResult() | RecordedBinary() | RecordedProbe(): - raise ReplayMiss( - f"recorded {interaction.request.method} {interaction.request.path} is not a streaming response" - ) - - -@dataclass(frozen=True, slots=True) -class ReplayTransport: - """A ``Transport`` served entirely from a recorded bundle: never opens a - connection, so a replay run cannot bill a provider.""" - - source: ReplaySource - master_key: str - - def bearer(self, key: str) -> AuthHeaders: - return AuthHeaders(authorization=f"Bearer {key}") - - @property - def master(self) -> AuthHeaders: - return self.bearer(self.master_key) - - def post[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction(recorded_request("post", path, headers=headers, body=json)) - ), - response_type, - ) - - def get[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - params: BaseModel, - response_type: type[R], - timeout: float | None = None, - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction(recorded_request("get", path, headers=headers, params=params)) - ), - response_type, - ) - - def delete[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - response_type: type[R], - params: BaseModel | None = None, - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction( - recorded_request("delete", path, headers=headers, body=json, params=params) - ) - ), - response_type, - ) - - def patch[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction(recorded_request("patch", path, headers=headers, body=json)) - ), - response_type, - ) - - def put[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction(recorded_request("put", path, headers=headers, body=json)) - ), - response_type, - ) - - def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: - return _expect_streaming( - self.source.next_interaction(recorded_request("stream", path, headers=headers, body=json)) - ) - - def stream_binary( - self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 - ) -> BinaryStream: - interaction = self.source.next_interaction( - recorded_request("stream_binary", path, headers=headers, body=json) - ) - match interaction.response: - case RecordedBinary(payload=payload): - return payload - case RecordedResult() | RecordedStreaming() | RecordedProbe(): - raise ReplayMiss( - f"recorded stream_binary {interaction.request.path} is not a binary stream" - ) - - def send( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - params: BaseModel | None = None, - stream: bool = False, - ) -> StreamingResponse: - return _expect_streaming( - self.source.next_interaction( - recorded_request("send", path, headers=headers, body=json, params=params) - ) - ) - - def probe(self, path: str, *, params: BaseModel) -> ProbeResult: - interaction = self.source.next_interaction( - recorded_request("probe", path, headers=self.master, params=params) - ) - match interaction.response: - case RecordedProbe(payload=payload): - return payload - case RecordedResult() | RecordedStreaming() | RecordedBinary(): - raise ReplayMiss(f"recorded probe {interaction.request.path} is not a probe result") - - def upload[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - form: BaseModel, - filename: str, - content: bytes, - file_content_type: str = "application/jsonl", - file_field: str = "file", - params: BaseModel | None = None, - response_type: type[R], - ) -> Result[R]: - return to_result( - _expect_result( - self.source.next_interaction( - recorded_request( - "upload", - path, - headers=headers, - params=params, - form=form, - file_name=filename, - file_content=content, - ) - ) - ), - response_type, - ) - - def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: - return _expect_streaming( - self.source.next_interaction(recorded_request("download", path, headers=headers)) - ) - - -@functools.lru_cache(maxsize=8) -def _shared_recorder(root: Path) -> BundleRecorder: - prepared = prepare_bundle(root) - if isinstance(prepared, UnsafeBundleDir): - raise ValueError(f"E2E_FIXTURE_DIR {prepared.path} {prepared.reason}") - return prepared - - -@functools.lru_cache(maxsize=8) -def _shared_replay_source(root: Path) -> ReplaySource: - loaded = load_bundle(root) - if isinstance(loaded, UnreadableBundle): - raise ValueError(f"cannot replay from {root}: {loaded.reason}") - return ReplaySource(bundle=loaded) - - -def replay_leftover_error(*, mode_raw: str, bundle_dir: Path, test_key: str) -> str | None: - """Teardown-time completeness check: in replay mode a passed test with - unconsumed recorded interactions must fail instead of passing against a - recording it no longer matches. Inert in every other mode.""" - if parse_fixture_mode(mode_raw) != "replay": - return None - return _shared_replay_source(bundle_dir).leftover_error(test_key) - - -def select_transport( - live: Transport, *, mode_raw: str, bundle_dir: Path, master_key: str -) -> Transport: - """The one seam every client build goes through: wraps (record), replaces - (replay), or passes through (live) the transport per E2E_FIXTURE_MODE. The - recorder and replay cursors are process-wide singletons per bundle dir, so - every client in a session shares one bundle and one recorded sequence.""" - mode = parse_fixture_mode(mode_raw) - match mode: - case InvalidFixtureMode(value=value): - raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") - case "live": - return live - case "record": - return RecordingTransport(inner=live, recorder=_shared_recorder(bundle_dir)) - case "replay": - return ReplayTransport(source=_shared_replay_source(bundle_dir), master_key=master_key) - case _: - assert_never(mode) - - -def fixture_mode_collection_error(mode_raw: str, bundle_dir: Path, *, now: datetime) -> str | None: - """Session-abort reason for a fixture-mode setup that can never work, or None. - Called at collection time (conftest pytest_sessionstart) so a stale or missing - bundle fails the whole run up front, naming the bundle age, instead of failing - every test individually.""" - mode = parse_fixture_mode(mode_raw) - match mode: - case InvalidFixtureMode(value=value): - return f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}" - case "live" | "record": - return None - case "replay": - freshness = check_freshness(bundle_dir, now=now) - match freshness: - case FreshBundle(): - return None - case StaleBundle(recorded_at=recorded_at, age=age, limit=limit): - return ( - f"fixture bundle at {bundle_dir} is stale: recorded {recorded_at.isoformat()}, " - f"age {format_age(age)} exceeds the {limit.days}-day limit; " - "re-record with E2E_FIXTURE_MODE=record" - ) - case UnreadableBundle(reason=reason): - return f"E2E_FIXTURE_MODE=replay cannot use bundle at {bundle_dir}: {reason}" - case _: - assert_never(freshness) - case _: - assert_never(mode) - - -def fixture_report_lines(mode_raw: str, bundle_dir: Path, *, now: datetime) -> list[str]: - """pytest report-header lines; empty in live mode so an unset - E2E_FIXTURE_MODE keeps today's output byte-identical.""" - mode = parse_fixture_mode(mode_raw) - match mode: - case InvalidFixtureMode() | "live": - return [] - case "record": - return [f"e2e fixture mode: record -> {bundle_dir}"] - case "replay": - freshness = check_freshness(bundle_dir, now=now) - match freshness: - case FreshBundle(manifest=manifest): - return [ - f"e2e fixture mode: replay <- {bundle_dir} " - f"(recorded {manifest.recorded_at.isoformat()}, harness {manifest.harness_version})" - ] - case StaleBundle() | UnreadableBundle(): - return [f"e2e fixture mode: replay <- {bundle_dir}"] - case _: - assert_never(freshness) - case _: - assert_never(mode) diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py new file mode 100644 index 00000000000..ab0791e6b74 --- /dev/null +++ b/tests/e2e/provider_edge.py @@ -0,0 +1,546 @@ +"""Provider-edge record/replay server for e2e runs (LIT-5745). + +Record and replay scope to provider-bound traffic only: the proxy boots for +real, tests hit it for real, and only the hop from the proxy to the provider +is recorded or served from a bundle. Suites opt in per deployment by pointing +``litellm_params.api_base`` at ``provider_edge_api_base(mount)``, which is an +in-process HTTP server mounting each supported provider under a path prefix +(``http://127.0.0.1:/openai`` forwards to ``https://api.openai.com``). +In record mode the edge relays each request verbatim, stores the interaction, +and serves the proxy the same filtered response replay will serve later; in +replay mode it serves straight from the bundle and never opens a provider +connection, so a green replay run with a fake provider key proves the entire +proxy pipeline (auth, routing, spend logging) without provider spend. + +Request identity reuses fixture_canonical.py: interactions match by canonical +content key, order-independent across keys and FIFO within one. Edge requests +store no headers at all: SDK telemetry headers vary run to run and credential +headers must never touch disk. An unmatched replay call returns HTTP +``REPLAY_MISS_STATUS`` naming the closest recorded interaction, which the +proxy relays as a provider error the failing test surfaces. + +v1 limits: only the mounts in ``EDGE_MOUNTS`` (SigV4 providers like Bedrock +sign the Host header, so a forwarding edge breaks their signatures), JSON and +opaque single-part bodies (multipart boundaries are random per request), +streaming fidelity is LIT-5742, and CI wiring is LIT-5748. Suites that do not +wire the edge keep hitting providers live in every mode. +""" + +from __future__ import annotations + +import base64 +import difflib +import functools +import hashlib +import threading +from collections import deque +from collections.abc import Mapping +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from itertools import islice +from pathlib import Path +from types import MappingProxyType +from typing import Final, Literal, assert_never +from urllib.parse import parse_qsl, urlsplit + +from pydantic import JsonValue, TypeAdapter + +from e2e_http import NetworkError, RawResponse, forward +from fixture_bundle import ( + BundleRecorder, + Interaction, + LoadedBundle, + RecordedHttpResponse, + RecordedRequest, + UnreadableBundle, + UnsafeBundleDir, + interaction_filename, + load_bundle, + prepare_bundle, + slug_for_test, +) +from fixture_canonical import CanonicalRequest, canonical_string, canonicalize +from fixture_mode import ( + FIXTURE_MODES, + InvalidFixtureMode, + ReplayMiss, + current_test_key, + parse_fixture_mode, +) + +EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( + { + "openai": "https://api.openai.com", + "anthropic": "https://api.anthropic.com", + } +) + +REPLAY_MISS_STATUS: Final = 599 + +_HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + } +) +_REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { + "host", + "content-length", + "accept-encoding", +} +_RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { + "content-encoding", + "content-length", + "set-cookie", +} + +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def _edge_request(method: str, path: str, query: str, body: bytes | None) -> RecordedRequest: + """The identity replay matches on: the edge path (mount included), the query + as params, and the body as parsed JSON, or as a canonicalized content digest + when it is not JSON so opaque uploads still match across runs.""" + params: Final = dict(parse_qsl(query, keep_blank_values=True)) + if not body: + return RecordedRequest(method=method.lower(), path=path, headers={}, params=params) + decoded: Final = body.decode("utf-8", errors="replace") + try: + parsed: Final[JsonValue] = _JSON.validate_json(decoded) + except ValueError: + return RecordedRequest( + method=method.lower(), + path=path, + headers={}, + params=params, + file_sha256=hashlib.sha256(canonical_string(decoded).encode()).hexdigest(), + file_bytes=len(body), + ) + return RecordedRequest(method=method.lower(), path=path, headers={}, params=params, body=parsed) + + +def _build_pool(recorded: tuple[Interaction, ...]) -> dict[str, deque[Interaction]]: + keys: Final = tuple(canonicalize(interaction.request).key for interaction in recorded) + return { + key: deque( + interaction + for candidate_key, interaction in zip(keys, recorded, strict=True) + if candidate_key == key + ) + for key in dict.fromkeys(keys) + } + + +def _closest_recorded( + canonical: CanonicalRequest, recorded: tuple[Interaction, ...] +) -> tuple[CanonicalRequest, str]: + candidates: Final = tuple(canonicalize(interaction.request) for interaction in recorded) + ratios: Final = tuple( + difflib.SequenceMatcher( + None, f"{canonical.method} {canonical.path}\n{canonical.content}", + f"{candidate.method} {candidate.path}\n{candidate.content}", + ).ratio() + for candidate in candidates + ) + best: Final = max(range(len(candidates)), key=lambda index: ratios[index]) + return candidates[best], interaction_filename(best, recorded[best].request) + + +def _miss_message(test_key: str, slug: str, canonical: CanonicalRequest, bundle: LoadedBundle) -> str: + recorded: Final = bundle.interactions.get(slug, ()) + if not recorded: + return ( + f"replay miss for {test_key}: computed key {canonical.key} but nothing is recorded " + f"under {slug}; re-record with E2E_FIXTURE_MODE=record" + ) + closest, closest_file = _closest_recorded(canonical, recorded) + diff: Final = "\n".join( + islice( + difflib.unified_diff( + closest.pretty_content().splitlines(), + canonical.pretty_content().splitlines(), + fromfile=f"closest recorded ({closest_file})", + tofile="test made", + lineterm="", + ), + 60, + ) + ) + return ( + f"replay miss for {test_key}: no recorded interaction matches key {canonical.key}; " + f"closest recorded key is {closest.key} ({closest_file})\n{diff}\n" + "re-record with E2E_FIXTURE_MODE=record" + ) + + +@dataclass(slots=True) +class ReplaySource: + """One shared pool per test over a loaded bundle, so every provider call the + proxy makes in the session consumes from the same recorded interactions. + Every pool is built once at construction and per-key consumption is a single + atomic deque pop, so concurrent replay calls never race. Calls match by + canonical content key: order-independent across distinct keys (concurrent + tests interleave calls nondeterministically), FIFO within one key (a retry + or poll loop replays its recorded responses in recorded order).""" + + bundle: LoadedBundle + _pools: dict[str, dict[str, deque[Interaction]]] = field(init=False) + + def __post_init__(self) -> None: + self._pools = { + slug: _build_pool(recorded) for slug, recorded in self.bundle.interactions.items() + } + + def _pool(self, slug: str) -> dict[str, deque[Interaction]]: + return self._pools.get(slug, {}) + + def next_interaction(self, request: RecordedRequest) -> Interaction: + test_key: Final = current_test_key() + slug: Final = slug_for_test(test_key) + pool: Final = self._pool(slug) + canonical: Final = canonicalize(request) + queue: Final = pool.get(canonical.key) + if queue is None: + raise ReplayMiss(_miss_message(test_key, slug, canonical, self.bundle)) + try: + return queue.popleft() + except IndexError: + raise ReplayMiss( + f"replay exhausted for {test_key}: every recorded interaction for key " + f"{canonical.key} is already consumed; re-record with E2E_FIXTURE_MODE=record" + ) from None + + def leftover_error(self, test_key: str) -> str | None: + """Non-None when the test consumed fewer interactions than were recorded, + meaning a passing replay proved less than the bundle claims.""" + slug: Final = slug_for_test(test_key) + recorded: Final = self.bundle.interactions.get(slug, ()) + if not recorded: + return None + leftover: Final = tuple( + interaction for queue in self._pool(slug).values() for interaction in queue + ) + if not leftover: + return None + return ( + f"replay incomplete for {test_key}: {len(leftover)} of {len(recorded)} recorded " + f"interactions never consumed, e.g. {canonicalize(leftover[0].request).key}; " + "re-record with E2E_FIXTURE_MODE=record" + ) + + +@dataclass(frozen=True, slots=True) +class RecordEdge: + """Record backend: forward to the provider, persist, serve the filtered copy. + The lock serializes recorder writes because the edge server handles requests + on concurrent threads.""" + + recorder: BundleRecorder + lock: threading.Lock + + +@dataclass(frozen=True, slots=True) +class ReplayEdge: + source: ReplaySource + + +type EdgeBackend = RecordEdge | ReplayEdge + + +@dataclass(frozen=True, slots=True) +class EdgeReply: + status_code: int + headers: dict[str, str] + body: bytes + + +def _text_reply(status_code: int, message: str) -> EdgeReply: + return EdgeReply( + status_code=status_code, + headers={"content-type": "text/plain; charset=utf-8"}, + body=message.encode(), + ) + + +def _reply_from_recorded(response: RecordedHttpResponse) -> EdgeReply: + return EdgeReply( + status_code=response.status_code, + headers=dict(response.headers), + body=base64.b64decode(response.body_b64), + ) + + +def _recorded_response(outcome: RawResponse | NetworkError) -> RecordedHttpResponse: + match outcome: + case RawResponse(status_code=status_code, headers=headers, body=body): + return RecordedHttpResponse( + status_code=status_code, + headers={ + name: value + for name, value in headers.items() + if name not in _RESPONSE_DROPPED_HEADERS + }, + body_b64=base64.b64encode(body).decode("ascii"), + ) + case NetworkError(message=message): + return RecordedHttpResponse( + status_code=502, + headers={"content-type": "text/plain; charset=utf-8"}, + body_b64=base64.b64encode( + f"provider edge could not reach the provider: {message}".encode() + ).decode("ascii"), + ) + + +def _upstream_url(upstream_base: str, upstream_path: str, query: str) -> str: + url: Final = f"{upstream_base}/{upstream_path}" + return f"{url}?{query}" if query else url + + +def _handle_record( + backend: RecordEdge, + request: RecordedRequest, + *, + method: str, + url: str, + headers: Mapping[str, str], + body: bytes | None, + timeout: float, +) -> EdgeReply: + forwarded: Final = { + name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS + } + outcome: Final = forward(method, url, headers=forwarded, body=body, timeout=timeout) + response: Final = _recorded_response(outcome) + with backend.lock: + backend.recorder.record(test_key=current_test_key(), request=request, response=response) + return _reply_from_recorded(response) + + +def _handle_replay(source: ReplaySource, request: RecordedRequest) -> EdgeReply: + try: + interaction: Final = source.next_interaction(request) + except ReplayMiss as miss: + return _text_reply(REPLAY_MISS_STATUS, str(miss)) + return _reply_from_recorded(interaction.response) + + +def handle_edge_request( + backend: EdgeBackend, + mounts: Mapping[str, str], + method: str, + raw_path: str, + headers: Mapping[str, str], + body: bytes | None, + *, + timeout: float, +) -> EdgeReply: + """The edge's pure core, one HTTP exchange in and out: resolve the mount + prefix, then record (forward + persist) or replay (serve from the bundle). + Socket-free so unit tests exercise every branch without a server.""" + split: Final = urlsplit(raw_path) + mount, _, upstream_path = split.path.lstrip("/").partition("/") + upstream_base: Final = mounts.get(mount) + if upstream_base is None: + return _text_reply( + 404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}" + ) + request: Final = _edge_request(method, split.path, split.query, body) + match backend: + case RecordEdge(): + return _handle_record( + backend, + request, + method=method, + url=_upstream_url(upstream_base, upstream_path, split.query), + headers=headers, + body=body, + timeout=timeout, + ) + case ReplayEdge(source=source): + return _handle_replay(source, request) + case _: + assert_never(backend) + + +class _EdgeHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + self._handle() + + def do_POST(self) -> None: + self._handle() + + def do_PUT(self) -> None: + self._handle() + + def do_PATCH(self) -> None: + self._handle() + + def do_DELETE(self) -> None: + self._handle() + + def _handle(self) -> None: + edge_server: Final = self.server + assert isinstance(edge_server, _EdgeHTTPServer) + length: Final = int(self.headers.get("content-length") or "0") + body: Final = self.rfile.read(length) if length else None + reply: Final = handle_edge_request( + edge_server.backend, + edge_server.mounts, + self.command, + self.path, + {name.lower(): value for name, value in self.headers.items()}, + body, + timeout=edge_server.forward_timeout, + ) + self.send_response(reply.status_code) + for name, value in reply.headers.items(): + self.send_header(name, value) + self.send_header("content-length", str(len(reply.body))) + self.end_headers() + self.wfile.write(reply.body) + + def log_message(self, format: str, *args: object) -> None: + """Silence the per-request stderr line BaseHTTPRequestHandler emits.""" + + +class _EdgeHTTPServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__( + self, + bind: tuple[str, int], + *, + backend: EdgeBackend, + mounts: Mapping[str, str], + forward_timeout: float, + ) -> None: + super().__init__(bind, _EdgeHandler) + self.backend: Final = backend + self.mounts: Final = mounts + self.forward_timeout: Final = forward_timeout + + +@dataclass(frozen=True, slots=True) +class ProviderEdge: + port: int + advertise_host: str + + def api_base(self, mount: str) -> str: + return f"http://{self.advertise_host}:{self.port}/{mount}" + + +@dataclass(frozen=True, slots=True) +class RunningEdge: + edge: ProviderEdge + server: _EdgeHTTPServer + + def shutdown(self) -> None: + self.server.shutdown() + self.server.server_close() + + +def start_provider_edge( + backend: EdgeBackend, + *, + mounts: Mapping[str, str] = EDGE_MOUNTS, + bind_host: str = "127.0.0.1", + advertise_host: str | None = None, + forward_timeout: float = 60.0, +) -> RunningEdge: + """Boot an edge server on an OS-assigned port in a daemon thread. + ``advertise_host`` is what api_base URLs name (it differs from the bind + host when the proxy runs in a container and reaches the host machine via + a gateway address like host.docker.internal).""" + server: Final = _EdgeHTTPServer( + (bind_host, 0), backend=backend, mounts=mounts, forward_timeout=forward_timeout + ) + thread: Final = threading.Thread(target=server.serve_forever, name="e2e-provider-edge", daemon=True) + thread.start() + return RunningEdge( + edge=ProviderEdge(port=server.server_address[1], advertise_host=advertise_host or bind_host), + server=server, + ) + + +@functools.lru_cache(maxsize=8) +def _shared_recorder(root: Path) -> BundleRecorder: + prepared = prepare_bundle(root) + if isinstance(prepared, UnsafeBundleDir): + raise ValueError(f"E2E_FIXTURE_DIR {prepared.path} {prepared.reason}") + return prepared + + +@functools.lru_cache(maxsize=8) +def _shared_replay_source(root: Path) -> ReplaySource: + loaded = load_bundle(root) + if isinstance(loaded, UnreadableBundle): + raise ValueError(f"cannot replay from {root}: {loaded.reason}") + return ReplaySource(bundle=loaded) + + +@functools.lru_cache(maxsize=8) +def _shared_edge( + mode: Literal["record", "replay"], + bundle_dir: Path, + bind_host: str, + advertise_host: str, + forward_timeout: float, +) -> ProviderEdge: + backend: Final[EdgeBackend] = ( + RecordEdge(recorder=_shared_recorder(bundle_dir), lock=threading.Lock()) + if mode == "record" + else ReplayEdge(source=_shared_replay_source(bundle_dir)) + ) + return start_provider_edge( + backend, + mounts=EDGE_MOUNTS, + bind_host=bind_host, + advertise_host=advertise_host, + forward_timeout=forward_timeout, + ).edge + + +def replay_leftover_error(*, mode_raw: str, bundle_dir: Path, test_key: str) -> str | None: + """Teardown-time completeness check: in replay mode a passed test with + unconsumed recorded interactions must fail instead of passing against a + recording it no longer matches. Inert in every other mode.""" + if parse_fixture_mode(mode_raw) != "replay": + return None + return _shared_replay_source(bundle_dir).leftover_error(test_key) + + +def provider_edge_api_base( + mount: str, + *, + mode_raw: str, + bundle_dir: Path, + bind_host: str, + advertise_host: str, + forward_timeout: float = 60.0, +) -> str | None: + """The api_base a suite gives an edge-wired deployment: None in live mode + (the deployment keeps its real provider api_base) and the process-wide edge + server's mount URL in record and replay, booting the server on first use.""" + mode: Final = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode(value=value): + raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") + case "live": + return None + case "record" | "replay": + if mount not in EDGE_MOUNTS: + raise ValueError( + f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(EDGE_MOUNTS))}" + ) + return _shared_edge(mode, bundle_dir, bind_host, advertise_host, forward_timeout).api_base(mount) + case _: + assert_never(mode) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 3cae337a5ff..6cdd3354bf7 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -65,8 +65,6 @@ from models import ( ) from e2e_config import ( CONTROL_PLANE_BASE_URL, - FIXTURE_DIR, - FIXTURE_MODE_RAW, MASTER_KEY, POLL_INTERVAL, POLL_TIMEOUT, @@ -74,7 +72,6 @@ from e2e_config import ( REQUEST_TIMEOUT, settle_propagation, ) -from fixture_transport import select_transport from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] @@ -547,9 +544,9 @@ def build_proxy_client( pass all three together, since a caller that overrides only the data plane would leave management calls pointed at the env default. - E2E_FIXTURE_MODE wraps (record) or replaces (replay) the transport here, so - every client built from this seam records or replays without changing shape; - unset it stays the plain SplitTransport (see fixture_transport.py).""" + Test-to-proxy traffic always goes over the wire, in every E2E_FIXTURE_MODE: + record and replay scope to the proxy's provider-bound calls via the + provider edge (see provider_edge.py), never to this transport.""" split = SplitTransport( data=HttpTransport( base_url=base_url, @@ -563,12 +560,7 @@ def build_proxy_client( ), ) return ProxyClient( - transport=select_transport( - split, - mode_raw=FIXTURE_MODE_RAW, - bundle_dir=FIXTURE_DIR, - master_key=master_key, - ), + transport=split, poll_timeout=POLL_TIMEOUT, poll_interval=POLL_INTERVAL, ) diff --git a/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py b/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py new file mode 100644 index 00000000000..ced7c819d42 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py @@ -0,0 +1,50 @@ +"""The provider-edge demonstrator: one spend-tracking flow wired through the +record/replay edge (LIT-5745). + +This is the reference for wiring a suite to the edge: register a deployment +whose ``api_base`` comes from ``e2e_config.provider_edge_base``, then exercise +the proxy exactly as a live test would. In live mode the base is None and the +deployment talks to the real provider; in record mode it talks through the +local edge, which forwards to the provider and captures the exchange; in +replay mode the same test drives the REAL proxy and REAL database on the +recorded provider traffic alone, so key auth, routing, and the spend-log +write path are all still under test with zero provider calls. +""" + +import pytest + +from e2e_config import CHEAP_OPENAI_MODEL, provider_edge_base +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from spend_e2e_client import SpendClient, unique_marker, unwrap + +pytestmark = pytest.mark.e2e + + +@pytest.mark.covers("quota_management.spend_tracking.chat_completions.logs_cost") +def test_edge_wired_chat_writes_nonzero_spend_row( + client: SpendClient, resources: ResourceManager, scoped_key: str +) -> None: + base = provider_edge_base("openai") + model = f"e2e-edge-openai-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=f"openai/{CHEAP_OPENAI_MODEL}", + api_key="os.environ/OPENAI_API_KEY", + api_base=None if base is None else f"{base}/v1", + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + chat = unwrap( + client.chat(scoped_key, model, f"reply with one word {unique_marker()}", max_tokens=16) + ) + assert chat.id + + rows = client.poll_logs_for_key( + scoped_key, predicate=lambda rs: any((r.spend or 0) > 0 for r in rs) + ) + matching = [row for row in rows if row.request_id == chat.id] + assert matching, f"no SpendLogs row for request_id {chat.id}; saw {len(rows)} row(s)" + assert (matching[0].spend or 0) > 0, f"spend row for {chat.id} has zero spend" diff --git a/tests/e2e/test_fixture_bundle.py b/tests/e2e/test_fixture_bundle.py index fd4cca6451f..b49ab565e39 100644 --- a/tests/e2e/test_fixture_bundle.py +++ b/tests/e2e/test_fixture_bundle.py @@ -1,9 +1,9 @@ -"""Harness coverage for the on-disk fixture bundle format (LIT-5729). +"""Harness coverage for the on-disk fixture bundle format (LIT-5729/LIT-5745). No proxy and no ``e2e`` marker: these pin the bundle CONTRACT - the seven-day freshness gate that names the bundle's age, record mode's wipe safety (never delete a directory that is not a bundle), collision-free per-test slugs, and -lossless Result round-trips - so replay can never silently drift from what +grouped-in-order loading - so replay can never silently drift from what record wrote. """ @@ -12,18 +12,6 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone from pathlib import Path -import pytest -from pydantic import BaseModel - -from e2e_http import ( - NetworkError, - RateLimitedError, - Result, - Success, - UnauthorizedError, - UnknownApiError, - ValidationError, -) from fixture_bundle import ( BUNDLE_FORMAT_VERSION, MANIFEST_FILENAME, @@ -32,28 +20,22 @@ from fixture_bundle import ( FreshBundle, LoadedBundle, Manifest, + RecordedHttpResponse, RecordedRequest, - RecordedResult, StaleBundle, UnreadableBundle, UnsafeBundleDir, check_freshness, format_age, - from_result, interaction_filename, load_bundle, prepare_bundle, slug_for_test, - to_result, ) NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) -class Payload(BaseModel): - value: str - - def write_manifest( root: Path, recorded_at: datetime, *, format_version: int = BUNDLE_FORMAT_VERSION ) -> None: @@ -74,20 +56,8 @@ def plain_request(path: str) -> RecordedRequest: return RecordedRequest(method="post", path=path, headers={}) -class TestResultRoundTrip: - @pytest.mark.parametrize( - "result", - [ - Success(status_code=201, data=Payload(value="ok")), - NetworkError(message="connection refused"), - UnauthorizedError(), - RateLimitedError(retry_after_seconds=7, body="slow down"), - ValidationError(message="bad shape"), - UnknownApiError(status_code=502, body="upstream exploded"), - ], - ) - def test_every_result_kind_survives_disk_and_back(self, result: Result[Payload]) -> None: - assert to_result(from_result(result), Payload) == result +def plain_response() -> RecordedHttpResponse: + return RecordedHttpResponse(status_code=401, headers={}, body_b64="") class TestFreshness: @@ -144,7 +114,7 @@ class TestPrepareBundle: prepared(root).record( test_key="old.py::test_old", request=plain_request("/stale"), - response=RecordedResult(kind="unauthorized"), + response=plain_response(), ) assert any(entry.is_dir() for entry in root.iterdir()) prepared(root) @@ -193,7 +163,7 @@ class TestRecordAndLoad: recorder.record( test_key=key, request=plain_request(path), - response=RecordedResult(kind="unauthorized"), + response=plain_response(), ) loaded = load_bundle(root) assert isinstance(loaded, LoadedBundle) @@ -208,7 +178,7 @@ class TestRecordAndLoad: recorder.record( test_key=key, request=plain_request(f"/{key[-3:]}"), - response=RecordedResult(kind="unauthorized"), + response=plain_response(), ) loaded = load_bundle(root) assert isinstance(loaded, LoadedBundle) diff --git a/tests/e2e/test_fixture_mode.py b/tests/e2e/test_fixture_mode.py new file mode 100644 index 00000000000..109bb9e1b11 --- /dev/null +++ b/tests/e2e/test_fixture_mode.py @@ -0,0 +1,114 @@ +"""Harness coverage for fixture-mode selection and determinism (LIT-5729/LIT-5745). + +No proxy and no ``e2e`` marker. Pins the mode parser, the deterministic +per-test marker sequence a replay run must regenerate, the collection-time +gate (including the stale message that names the bundle's age), and the pytest +report header. The provider-edge record/replay behavior itself is pinned in +test_provider_edge.py. +""" + +from __future__ import annotations + +import hashlib +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from fixture_bundle import BUNDLE_FORMAT_VERSION, MANIFEST_FILENAME, Manifest +from fixture_mode import ( + InvalidFixtureMode, + current_test_key, + deterministic_marker, + fixture_mode_collection_error, + fixture_report_lines, + parse_fixture_mode, +) + +NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) + + +def write_manifest(root: Path, recorded_at: datetime) -> None: + root.mkdir(parents=True, exist_ok=True) + manifest = Manifest( + format_version=BUNDLE_FORMAT_VERSION, recorded_at=recorded_at, harness_version="abc1234" + ) + (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8") + + +class TestParseFixtureMode: + @pytest.mark.parametrize( + ("raw", "expected"), + [("live", "live"), ("record", "record"), ("replay", "replay"), ("", "live"), (" REPLAY ", "replay")], + ) + def test_known_values_normalize(self, raw: str, expected: str) -> None: + assert parse_fixture_mode(raw) == expected + + def test_unknown_value_is_invalid_with_the_original_spelling(self) -> None: + assert parse_fixture_mode("cached") == InvalidFixtureMode(value="cached") + + +class TestDeterministicMarker: + def test_sequence_is_a_pure_function_of_test_and_ordinal(self) -> None: + """A replay process must regenerate exactly the markers the record + process generated, so the Nth marker of a test is pinned to a pure + function of the node id and N.""" + key = current_test_key() + assert deterministic_marker() == hashlib.sha1(f"{key}#0".encode()).hexdigest()[:12] + assert deterministic_marker() == hashlib.sha1(f"{key}#1".encode()).hexdigest()[:12] + + +class TestCurrentTestKey: + def test_names_this_test_and_strips_the_phase(self) -> None: + key = current_test_key() + assert key.endswith("TestCurrentTestKey::test_names_this_test_and_strips_the_phase") + assert "(call)" not in key + + +class TestCollectionGate: + def test_invalid_mode_names_the_value_and_the_choices(self, tmp_path: Path) -> None: + assert ( + fixture_mode_collection_error("cached", tmp_path, now=NOW) + == "E2E_FIXTURE_MODE='cached' is not one of live, record, replay" + ) + + @pytest.mark.parametrize("mode_raw", ["live", "", "record"]) + def test_live_and_record_never_block_collection(self, mode_raw: str, tmp_path: Path) -> None: + assert fixture_mode_collection_error(mode_raw, tmp_path / "missing", now=NOW) is None + + def test_replay_with_no_bundle_says_how_to_record_one(self, tmp_path: Path) -> None: + reason = fixture_mode_collection_error("replay", tmp_path / "missing", now=NOW) + assert reason is not None + assert f"no {MANIFEST_FILENAME}" in reason + assert "E2E_FIXTURE_MODE=record" in reason + + def test_stale_replay_bundle_fails_naming_its_age(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=9, hours=5)) + reason = fixture_mode_collection_error("replay", root, now=NOW) + assert reason is not None + assert "age 9d5h exceeds the 7-day limit" in reason + assert "re-record with E2E_FIXTURE_MODE=record" in reason + + def test_fresh_replay_bundle_collects(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=2)) + assert fixture_mode_collection_error("replay", root, now=NOW) is None + + +class TestReportHeader: + def test_live_mode_prints_nothing(self, tmp_path: Path) -> None: + assert fixture_report_lines("live", tmp_path, now=NOW) == [] + assert fixture_report_lines("", tmp_path, now=NOW) == [] + + def test_record_and_replay_name_the_bundle(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorded_at = NOW - timedelta(days=1) + write_manifest(root, recorded_at) + assert fixture_report_lines("record", root, now=NOW) == [ + f"e2e fixture mode: record -> {root}" + ] + replay_lines = fixture_report_lines("replay", root, now=NOW) + assert len(replay_lines) == 1 + assert "replay" in replay_lines[0] + assert recorded_at.isoformat() in replay_lines[0] diff --git a/tests/e2e/test_fixture_transport.py b/tests/e2e/test_fixture_transport.py deleted file mode 100644 index e61088d841c..00000000000 --- a/tests/e2e/test_fixture_transport.py +++ /dev/null @@ -1,676 +0,0 @@ -"""Harness coverage for the record/replay transports (LIT-5729). - -No proxy and no ``e2e`` marker. A fake in-memory ``Transport`` stands in for -the live one (dependency injection, no monkeypatching): recording must pass -every value through unchanged while writing one redacted interaction file per -call, and replay must serve identical values from the bundle alone - the -fake's call log proves nothing reaches the inner transport - failing hard -(``ReplayMiss``) on any content drift, printing the computed canonical key and -the closest recorded key (LIT-5741; the pure canonicalizer is pinned in -test_fixture_canonical.py). The collection-time gate and report header are -pinned here too, including the stale message that names the bundle's age. -""" - -from __future__ import annotations - -import hashlib -import sys -import threading -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field -from datetime import datetime, timedelta, timezone -from pathlib import Path -from uuid import uuid4 - -import pytest -from pydantic import BaseModel - -from e2e_http import ( - AuthHeaders, - BinaryStream, - ProbeResult, - Result, - StreamingResponse, - Success, -) -from fixture_bundle import ( - BUNDLE_FORMAT_VERSION, - MANIFEST_FILENAME, - BundleRecorder, - Interaction, - LoadedBundle, - Manifest, - RecordedResult, - load_bundle, - prepare_bundle, - slug_for_test, -) -from fixture_canonical import canonicalize -from fixture_transport import ( - InvalidFixtureMode, - RecordingTransport, - ReplayMiss, - ReplaySource, - ReplayTransport, - current_test_key, - deterministic_marker, - fixture_mode_collection_error, - fixture_report_lines, - parse_fixture_mode, - recorded_request, - replay_leftover_error, - select_transport, -) -from transport import Transport - -NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) - - -class Payload(BaseModel): - value: str - - -class Body(BaseModel): - prompt: str - - -class Query(BaseModel): - q: str - - -class DeployParams(BaseModel): - model: str - api_key: str | None = None - aws_secret_access_key: str | None = None - - -class DeployBody(BaseModel): - model_name: str - litellm_params: DeployParams - - -STREAMING = StreamingResponse( - status_code=200, - body="", - content_type="text/event-stream", - chunks=2, - stream_events=["one", "two"], - stream_done=True, -) -BINARY = BinaryStream(status_code=200, content_type="audio/mpeg", chunk_count=3, total_bytes=42) -PROBE = ProbeResult(status_code=200, body="alive") - - -@dataclass -class FakeTransport: - calls: list[str] = field(default_factory=list) - - def bearer(self, key: str) -> AuthHeaders: - return AuthHeaders(authorization=f"Bearer {key}") - - @property - def master(self) -> AuthHeaders: - return self.bearer("sk-fake-master") - - def _success[R: BaseModel](self, response_type: type[R]) -> Result[R]: - return Success(status_code=200, data=response_type.model_validate({"value": "live"})) - - def post[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - self.calls.append(f"post {path}") - return self._success(response_type) - - def get[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - params: BaseModel, - response_type: type[R], - timeout: float | None = None, - ) -> Result[R]: - self.calls.append(f"get {path}") - return self._success(response_type) - - def delete[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - response_type: type[R], - params: BaseModel | None = None, - ) -> Result[R]: - self.calls.append(f"delete {path}") - return self._success(response_type) - - def patch[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - self.calls.append(f"patch {path}") - return self._success(response_type) - - def put[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] - ) -> Result[R]: - self.calls.append(f"put {path}") - return self._success(response_type) - - def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: - self.calls.append(f"stream {path}") - return STREAMING - - def stream_binary( - self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 - ) -> BinaryStream: - self.calls.append(f"stream_binary {path}") - return BINARY - - def send( - self, - path: str, - *, - headers: BaseModel, - json: BaseModel, - params: BaseModel | None = None, - stream: bool = False, - ) -> StreamingResponse: - self.calls.append(f"send {path}") - return STREAMING - - def probe(self, path: str, *, params: BaseModel) -> ProbeResult: - self.calls.append(f"probe {path}") - return PROBE - - def upload[R: BaseModel]( - self, - path: str, - *, - headers: BaseModel, - form: BaseModel, - filename: str, - content: bytes, - file_content_type: str = "application/jsonl", - file_field: str = "file", - params: BaseModel | None = None, - response_type: type[R], - ) -> Result[R]: - self.calls.append(f"upload {path}") - return self._success(response_type) - - def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: - self.calls.append(f"download {path}") - return STREAMING - - -def make_recorder(root: Path) -> BundleRecorder: - recorder = prepare_bundle(root) - assert isinstance(recorder, BundleRecorder) - return recorder - - -def replay_source(root: Path) -> ReplaySource: - loaded = load_bundle(root) - assert isinstance(loaded, LoadedBundle) - return ReplaySource(bundle=loaded) - - -def this_tests_files(root: Path) -> list[Path]: - slug_dir = root / slug_for_test(current_test_key()) - return sorted(slug_dir.glob("*.json")) if slug_dir.is_dir() else [] - - -def write_manifest(root: Path, recorded_at: datetime) -> None: - root.mkdir(parents=True, exist_ok=True) - manifest = Manifest( - format_version=BUNDLE_FORMAT_VERSION, recorded_at=recorded_at, harness_version="abc1234" - ) - (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8") - - -class TestParseFixtureMode: - @pytest.mark.parametrize( - ("raw", "expected"), - [("live", "live"), ("record", "record"), ("replay", "replay"), ("", "live"), (" REPLAY ", "replay")], - ) - def test_known_values_normalize(self, raw: str, expected: str) -> None: - assert parse_fixture_mode(raw) == expected - - def test_unknown_value_is_invalid_with_the_original_spelling(self) -> None: - assert parse_fixture_mode("cached") == InvalidFixtureMode(value="cached") - - -class TestDeterministicMarker: - def test_sequence_is_a_pure_function_of_test_and_ordinal(self) -> None: - """A replay process must regenerate exactly the markers the record - process generated, so the Nth marker of a test is pinned to a pure - function of the node id and N.""" - key = current_test_key() - assert deterministic_marker() == hashlib.sha1(f"{key}#0".encode()).hexdigest()[:12] - assert deterministic_marker() == hashlib.sha1(f"{key}#1".encode()).hexdigest()[:12] - - -class TestCurrentTestKey: - def test_names_this_test_and_strips_the_phase(self) -> None: - key = current_test_key() - assert key.endswith("TestCurrentTestKey::test_names_this_test_and_strips_the_phase") - assert "(call)" not in key - - -class TestRecordingTransport: - def test_passes_the_result_through_and_writes_one_file_per_call(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - result = recording.post( - "/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload - ) - assert result == Success(status_code=200, data=Payload(value="live")) - assert fake.calls == ["post /model/new"] - files = this_tests_files(root) - assert [file.name for file in files] == ["0000-post-model-new.json"] - interaction = Interaction.model_validate_json(files[0].read_text(encoding="utf-8")) - assert interaction.request.method == "post" - assert interaction.request.path == "/model/new" - - def test_redacts_auth_header_values_in_the_recorded_request(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - headers = AuthHeaders.model_validate( - {"authorization": "Bearer sk-secret", "x-litellm-api-key": "sk-other"} - ) - recording.post("/key/generate", headers=headers, json=Body(prompt="x"), response_type=Payload) - interaction = Interaction.model_validate_json( - this_tests_files(root)[0].read_text(encoding="utf-8") - ) - assert interaction.request.headers == { - "authorization": "", - "x-litellm-api-key": "", - } - assert "sk-secret" not in this_tests_files(root)[0].read_text(encoding="utf-8") - - def test_redacts_credential_body_fields_in_the_recorded_request(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post( - "/model/new", - headers=fake.master, - json=DeployBody( - model_name="m", - litellm_params=DeployParams(model="openai/gpt", api_key="sk-live-provider-secret-123456"), - ), - response_type=Payload, - ) - raw = this_tests_files(root)[0].read_text(encoding="utf-8") - interaction = Interaction.model_validate_json(raw) - assert "sk-live-provider-secret-123456" not in raw - assert isinstance(interaction.request.body, dict) - params = interaction.request.body["litellm_params"] - assert isinstance(params, dict) - assert params["api_key"] == "" - assert params["aws_secret_access_key"] is None - - def test_upload_records_a_content_digest_not_the_bytes(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.upload( - "/v1/files", - headers=fake.master, - form=Query(q="batch"), - filename="batch.jsonl", - content=b'{"custom_id": "1"}', - response_type=Payload, - ) - interaction = Interaction.model_validate_json( - this_tests_files(root)[0].read_text(encoding="utf-8") - ) - assert interaction.request.file_name == "batch.jsonl" - assert interaction.request.file_bytes == len(b'{"custom_id": "1"}') - assert interaction.request.file_sha256 is not None - assert "custom_id" not in interaction.request.model_dump_json() - - -class TestReplayTransport: - def test_serves_recorded_values_without_touching_the_inner_transport( - self, tmp_path: Path - ) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recorded_post = recording.post( - "/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload - ) - recorded_get = recording.get( - "/v1/models", headers=fake.master, params=Query(q="all"), response_type=Payload - ) - recorded_stream = recording.stream( - "/chat/completions", headers=fake.master, json=Body(prompt="hi") - ) - recorded_probe = recording.probe("/health/liveliness", params=Query(q="1")) - recorded_binary = recording.stream_binary( - "/v1/audio/speech", headers=fake.master, json=Body(prompt="say") - ) - calls_after_record = list(fake.calls) - - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - assert ( - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - == recorded_post - ) - assert ( - replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) - == recorded_get - ) - assert ( - replay.stream("/chat/completions", headers=replay.master, json=Body(prompt="hi")) - == recorded_stream - ) - assert replay.probe("/health/liveliness", params=Query(q="1")) == recorded_probe - assert ( - replay.stream_binary("/v1/audio/speech", headers=replay.master, json=Body(prompt="say")) - == recorded_binary - ) - assert fake.calls == calls_after_record - - def test_miss_names_the_computed_key_and_the_closest_recorded_key(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - with pytest.raises(ReplayMiss) as excinfo: - replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) - message = str(excinfo.value) - assert "no recorded interaction matches key get /v1/models #" in message - assert "closest recorded key is post /model/new #" in message - assert "0000-post-model-new.json" in message - assert "re-record with E2E_FIXTURE_MODE=record" in message - - def test_content_drift_on_the_same_route_misses_with_no_live_call(self, tmp_path: Path) -> None: - """The naive verb+path match replayed a stale response for a request - whose content had changed, silently passing; a content key must miss, - print both canonical forms' diff, and never reach the inner transport.""" - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - calls_after_record = list(fake.calls) - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - with pytest.raises(ReplayMiss) as excinfo: - replay.post("/model/new", headers=replay.master, json=Body(prompt="y"), response_type=Payload) - message = str(excinfo.value) - assert "no recorded interaction matches key post /model/new #" in message - assert "closest recorded key is post /model/new #" in message - assert '- "prompt": "x"' in message - assert '+ "prompt": "y"' in message - assert fake.calls == calls_after_record - - def test_exhausted_key_names_the_key(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - with pytest.raises( - ReplayMiss, match=r"every recorded interaction for key post /model/new #\w{16} is already consumed" - ): - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - - def test_replays_out_of_recorded_order_across_distinct_keys(self, tmp_path: Path) -> None: - """Concurrent tests interleave independent calls nondeterministically - (e.g. a burst of parallel chat calls), so replay matches by content, - never by recorded position.""" - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - recording.post("/key/generate", headers=fake.master, json=Body(prompt="k"), response_type=Payload) - source = replay_source(root) - replay: Transport = ReplayTransport(source=source, master_key="sk-1234") - replay.post("/key/generate", headers=replay.master, json=Body(prompt="k"), response_type=Payload) - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - assert source.leftover_error(current_test_key()) is None - - def test_identical_requests_replay_their_responses_in_recorded_order(self, tmp_path: Path) -> None: - """A poll loop makes the same request repeatedly and asserts on the - progression, so duplicates under one key stay FIFO.""" - root = tmp_path / "bundle" - recorder = make_recorder(root) - recorder.record( - test_key=current_test_key(), - request=recorded_request( - "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") - ), - response=RecordedResult(kind="success", status_code=200, data={"value": "first"}), - ) - recorder.record( - test_key=current_test_key(), - request=recorded_request( - "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") - ), - response=RecordedResult(kind="success", status_code=200, data={"value": "second"}), - ) - replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") - first = replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) - second = replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) - assert first == Success(status_code=200, data=Payload(value="first")) - assert second == Success(status_code=200, data=Payload(value="second")) - - def test_concurrent_replays_of_one_key_serve_each_recording_exactly_once(self, tmp_path: Path) -> None: - """A burst of parallel identical calls consumes one shared pool: no - response duplicated, none forgotten, nothing left over at teardown. - The tiny switch interval forces thread preemption inside pool setup - and consumption, so a non-atomic pool build or pop fails this test.""" - root = tmp_path / "bundle" - recorder = make_recorder(root) - for ordinal in range(32): - recorder.record( - test_key=current_test_key(), - request=recorded_request( - "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") - ), - response=RecordedResult(kind="success", status_code=200, data={"value": f"v{ordinal:02d}"}), - ) - source = replay_source(root) - replay: Transport = ReplayTransport(source=source, master_key="sk-1234") - barrier = threading.Barrier(8) - - def consume_one() -> str: - result = replay.get( - "/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload - ) - assert isinstance(result, Success) - return result.data.value - - def consume(_: int) -> tuple[str, ...]: - barrier.wait() - return tuple(consume_one() for _call in range(4)) - - previous_interval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) - try: - with ThreadPoolExecutor(max_workers=8) as executor: - served = sorted(value for values in executor.map(consume, range(8)) for value in values) - finally: - sys.setswitchinterval(previous_interval) - assert served == [f"v{ordinal:02d}" for ordinal in range(32)] - assert source.leftover_error(current_test_key()) is None - - -class TestRecordedKeySets: - def test_two_separate_recordings_of_one_flow_produce_identical_key_sets( - self, tmp_path: Path - ) -> None: - """Everything a run randomizes (markers, virtual keys, dates) must - canonicalize out, so separately recorded runs of the same suite agree - on every match key and a bundle recorded elsewhere replays here.""" - - def record_flow(root: Path, run_date: str) -> list[str]: - fake = FakeTransport() - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - marker = deterministic_marker() - recording.post( - "/model/new", - headers=fake.master, - json=DeployBody( - model_name=f"e2e-chat-{marker}", - litellm_params=DeployParams(model="openai/gpt", api_key=f"sk-live-{uuid4().hex}"), - ), - response_type=Payload, - ) - recording.post( - "/chat/completions", - headers=recording.bearer(f"sk-{uuid4().hex}"), - json=Body(prompt=f"Reply with the single word ok. {marker}"), - response_type=Payload, - ) - recording.get( - "/spend/logs", headers=fake.master, params=Query(q=run_date), response_type=Payload - ) - loaded = load_bundle(root) - assert isinstance(loaded, LoadedBundle) - return sorted( - canonicalize(interaction.request).key - for interactions in loaded.interactions.values() - for interaction in interactions - ) - - first_keys = record_flow(tmp_path / "one", "2026-08-18") - second_keys = record_flow(tmp_path / "two", "2026-08-19") - assert first_keys == second_keys - assert len(first_keys) == 3 - - -class TestReplayLeftover: - def test_fully_consumed_recording_leaves_nothing(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - source = replay_source(root) - replay: Transport = ReplayTransport(source=source, master_key="sk-1234") - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - assert source.leftover_error(current_test_key()) is None - - def test_unconsumed_trailing_interactions_name_the_next_call(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - recording.probe("/health/liveliness", params=Query(q="1")) - source = replay_source(root) - replay: Transport = ReplayTransport(source=source, master_key="sk-1234") - replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) - error = source.leftover_error(current_test_key()) - assert error is not None - assert "1 of 2 recorded interactions never consumed" in error - assert "e.g. probe /health/liveliness #" in error - assert "re-record with E2E_FIXTURE_MODE=record" in error - - def test_test_without_recordings_has_no_leftover(self, tmp_path: Path) -> None: - root = tmp_path / "bundle" - make_recorder(root) - assert replay_source(root).leftover_error("suite.py::test_never_recorded") is None - - def test_inert_outside_replay_mode(self, tmp_path: Path) -> None: - missing = tmp_path / "missing" - assert replay_leftover_error(mode_raw="", bundle_dir=missing, test_key="k") is None - assert replay_leftover_error(mode_raw="record", bundle_dir=missing, test_key="k") is None - - def test_replay_mode_reads_the_shared_bundle(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) - recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) - error = replay_leftover_error(mode_raw="replay", bundle_dir=root, test_key=current_test_key()) - assert error is not None - assert "1 of 1 recorded interactions never consumed" in error - - -class TestSelectTransport: - def test_live_returns_the_live_transport_untouched(self, tmp_path: Path) -> None: - fake = FakeTransport() - for mode_raw in ("live", ""): - assert ( - select_transport(fake, mode_raw=mode_raw, bundle_dir=tmp_path / "b", master_key="sk") - is fake - ) - - def test_record_wraps_live_and_starts_a_fresh_bundle(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - write_manifest(root, NOW - timedelta(days=30)) - (root / "old-test-slug").mkdir() - (root / "old-test-slug" / "0000-post-old.json").write_text("{}", encoding="utf-8") - selected = select_transport(fake, mode_raw="record", bundle_dir=root, master_key="sk") - assert isinstance(selected, RecordingTransport) - assert selected.inner is fake - assert {entry.name for entry in root.iterdir()} == {MANIFEST_FILENAME} - - def test_replay_builds_a_transport_from_the_bundle_alone(self, tmp_path: Path) -> None: - fake = FakeTransport() - root = tmp_path / "bundle" - make_recorder(root) - selected = select_transport(fake, mode_raw="replay", bundle_dir=root, master_key="sk-master") - assert isinstance(selected, ReplayTransport) - assert selected.master == AuthHeaders(authorization="Bearer sk-master") - - def test_invalid_mode_raises_naming_the_value(self, tmp_path: Path) -> None: - with pytest.raises(ValueError, match="cached"): - select_transport( - FakeTransport(), mode_raw="cached", bundle_dir=tmp_path / "b", master_key="sk" - ) - - -class TestCollectionGate: - def test_invalid_mode_names_the_value_and_the_choices(self, tmp_path: Path) -> None: - assert ( - fixture_mode_collection_error("cached", tmp_path, now=NOW) - == "E2E_FIXTURE_MODE='cached' is not one of live, record, replay" - ) - - @pytest.mark.parametrize("mode_raw", ["live", "", "record"]) - def test_live_and_record_never_block_collection(self, mode_raw: str, tmp_path: Path) -> None: - assert fixture_mode_collection_error(mode_raw, tmp_path / "missing", now=NOW) is None - - def test_replay_with_no_bundle_says_how_to_record_one(self, tmp_path: Path) -> None: - reason = fixture_mode_collection_error("replay", tmp_path / "missing", now=NOW) - assert reason is not None - assert f"no {MANIFEST_FILENAME}" in reason - assert "E2E_FIXTURE_MODE=record" in reason - - def test_stale_replay_bundle_fails_naming_its_age(self, tmp_path: Path) -> None: - root = tmp_path / "bundle" - write_manifest(root, NOW - timedelta(days=9, hours=5)) - reason = fixture_mode_collection_error("replay", root, now=NOW) - assert reason is not None - assert "age 9d5h exceeds the 7-day limit" in reason - assert "re-record with E2E_FIXTURE_MODE=record" in reason - - def test_fresh_replay_bundle_collects(self, tmp_path: Path) -> None: - root = tmp_path / "bundle" - write_manifest(root, NOW - timedelta(days=2)) - assert fixture_mode_collection_error("replay", root, now=NOW) is None - - -class TestReportHeader: - def test_live_mode_prints_nothing(self, tmp_path: Path) -> None: - assert fixture_report_lines("live", tmp_path, now=NOW) == [] - assert fixture_report_lines("", tmp_path, now=NOW) == [] - - def test_record_and_replay_name_the_bundle(self, tmp_path: Path) -> None: - root = tmp_path / "bundle" - recorded_at = NOW - timedelta(days=1) - write_manifest(root, recorded_at) - assert fixture_report_lines("record", root, now=NOW) == [ - f"e2e fixture mode: record -> {root}" - ] - replay_lines = fixture_report_lines("replay", root, now=NOW) - assert len(replay_lines) == 1 - assert "replay" in replay_lines[0] - assert recorded_at.isoformat() in replay_lines[0] diff --git a/tests/e2e/test_provider_edge.py b/tests/e2e/test_provider_edge.py new file mode 100644 index 00000000000..492eee57aaf --- /dev/null +++ b/tests/e2e/test_provider_edge.py @@ -0,0 +1,492 @@ +"""Harness coverage for the provider-edge record/replay server (LIT-5745). + +No proxy and no ``e2e`` marker. A stdlib http.server stands in for the +provider (dependency injection via the mounts mapping, no monkeypatching): +record mode must forward each edge call to it verbatim, persist one +interaction file, and serve the proxy the same filtered response replay will +serve later; replay mode must serve byte-identical responses from the bundle +alone, with the fake provider's hit log proving nothing leaves the process, +and answer any drifted call with HTTP ``REPLAY_MISS_STATUS`` naming the +computed and closest recorded canonical keys (LIT-5741; the pure canonicalizer +is pinned in test_fixture_canonical.py). Requests are made through +``e2e_http.forward`` so the whole HTTP surface of the edge is exercised; the +pure ``handle_edge_request`` core is pinned socket-free alongside. +""" + +from __future__ import annotations + +import base64 +import json +import threading +from collections.abc import Generator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from pydantic import TypeAdapter + +from e2e_http import RawResponse, forward +from fixture_bundle import ( + BundleRecorder, + Interaction, + LoadedBundle, + RecordedHttpResponse, + RecordedRequest, + load_bundle, + prepare_bundle, + slug_for_test, +) +from fixture_mode import current_test_key +from provider_edge import ( + REPLAY_MISS_STATUS, + EdgeBackend, + ProviderEdge, + RecordEdge, + ReplayEdge, + ReplaySource, + handle_edge_request, + provider_edge_api_base, + replay_leftover_error, + start_provider_edge, +) + +CHAT_PATH = "/openai/v1/chat/completions" +REPLAY_MOUNTS = {"openai": "https://replay.invalid"} +JSON_OBJECT = TypeAdapter(dict[str, object]) + + +def json_object(body: bytes) -> dict[str, object]: + return JSON_OBJECT.validate_json(body) + + +class _FakeProvider(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, bind: tuple[str, int]) -> None: + super().__init__(bind, _FakeProviderHandler) + self.hits: list[str] = [] + + +class _FakeProviderHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + self._respond() + + def do_GET(self) -> None: + self._respond() + + def _respond(self) -> None: + provider = self.server + assert isinstance(provider, _FakeProvider) + length = int(self.headers.get("content-length") or "0") + body = self.rfile.read(length) if length else b"" + provider.hits.append(f"{self.command} {self.path}") + payload = json.dumps( + {"echo": body.decode("utf-8"), "path": self.path, "hit": len(provider.hits)} + ).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(payload))) + self.send_header("x-upstream", "fake") + self.send_header("set-cookie", "session=fake-cookie") + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format: str, *args: object) -> None: + """Silence the per-request stderr line BaseHTTPRequestHandler emits.""" + + +@contextmanager +def fake_provider() -> Generator[_FakeProvider]: + server = _FakeProvider(("127.0.0.1", 0)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + + +def provider_url(server: _FakeProvider) -> str: + return f"http://127.0.0.1:{server.server_address[1]}" + + +@contextmanager +def running_edge(backend: EdgeBackend, mounts: Mapping[str, str]) -> Generator[ProviderEdge]: + running = start_provider_edge(backend, mounts=mounts, bind_host="127.0.0.1") + try: + yield running.edge + finally: + running.shutdown() + + +def record_backend(root: Path) -> RecordEdge: + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + return RecordEdge(recorder=recorder, lock=threading.Lock()) + + +def replay_source(root: Path) -> ReplaySource: + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + return ReplaySource(bundle=loaded) + + +def call_edge( + edge: ProviderEdge, + method: str, + path: str, + *, + body: bytes | None = None, + headers: dict[str, str] | None = None, +) -> RawResponse: + outcome = forward( + method, + f"http://{edge.advertise_host}:{edge.port}{path}", + headers=headers or {}, + body=body, + timeout=10.0, + ) + assert isinstance(outcome, RawResponse) + return outcome + + +def this_tests_files(root: Path) -> list[Path]: + slug_dir = root / slug_for_test(current_test_key()) + return sorted(slug_dir.glob("*.json")) if slug_dir.is_dir() else [] + + +def chat_body(prompt: str) -> bytes: + return json.dumps({"model": "gpt", "messages": [{"role": "user", "content": prompt}]}).encode() + + +class TestRecordMode: + def test_forwards_to_the_provider_and_writes_one_interaction_file(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + reply = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert provider.hits == ["POST /v1/chat/completions"] + assert reply.status_code == 200 + served = json_object(reply.body) + assert served["echo"] == chat_body("hi").decode() + files = this_tests_files(root) + assert [file.name for file in files] == ["0000-post-openai-v1-chat-completions.json"] + interaction = Interaction.model_validate_json(files[0].read_text(encoding="utf-8")) + assert interaction.request.method == "post" + assert interaction.request.path == CHAT_PATH + assert interaction.request.body == json_object(chat_body("hi")) + assert interaction.response.status_code == 200 + + def test_never_stores_headers_so_credentials_never_touch_disk(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge( + edge, + "POST", + CHAT_PATH, + body=chat_body("hi"), + headers={"authorization": "Bearer sk-live-provider-secret-abc123"}, + ) + raw = this_tests_files(root)[0].read_text(encoding="utf-8") + assert "sk-live-provider-secret-abc123" not in raw + interaction = Interaction.model_validate_json(raw) + assert interaction.request.headers == {} + + def test_strips_volatile_response_headers_and_serves_the_filtered_copy(self, tmp_path: Path) -> None: + """What record serves the proxy must equal what replay will serve later + (record/replay parity), so the filtered stored copy is served in both.""" + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + reply = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert reply.headers.get("x-upstream") == "fake" + assert "set-cookie" not in reply.headers + interaction = Interaction.model_validate_json( + this_tests_files(root)[0].read_text(encoding="utf-8") + ) + assert interaction.response.headers.get("x-upstream") == "fake" + assert "set-cookie" not in interaction.response.headers + assert "content-length" not in interaction.response.headers + + def test_unreachable_provider_records_and_serves_a_502(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with running_edge(record_backend(root), {"openai": "http://127.0.0.1:9"}) as edge: + reply = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert reply.status_code == 502 + assert b"could not reach the provider" in reply.body + interaction = Interaction.model_validate_json( + this_tests_files(root)[0].read_text(encoding="utf-8") + ) + assert interaction.response.status_code == 502 + + +class TestReplayMode: + def test_serves_recorded_bytes_with_zero_provider_hits(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + recorded = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + hits_after_record = list(provider.hits) + with running_edge( + ReplayEdge(source=replay_source(root)), {"openai": provider_url(provider)} + ) as edge: + replayed = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert provider.hits == hits_after_record + assert replayed.status_code == recorded.status_code + assert replayed.body == recorded.body + assert replayed.headers.get("x-upstream") == "fake" + + def test_request_identity_ignores_auth_headers(self, tmp_path: Path) -> None: + """The proxy sends different bearer tokens across runs (fresh virtual + keys, rotated provider keys), so headers are no part of the match.""" + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge( + edge, "POST", CHAT_PATH, body=chat_body("hi"), + headers={"authorization": "Bearer sk-first-run"}, + ) + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + replayed = call_edge( + edge, "POST", CHAT_PATH, body=chat_body("hi"), + headers={"authorization": "Bearer sk-second-run"}, + ) + assert replayed.status_code == 200 + + def test_content_drift_returns_the_miss_status_naming_both_keys(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("x")) + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + missed = call_edge(edge, "POST", CHAT_PATH, body=chat_body("y")) + assert missed.status_code == REPLAY_MISS_STATUS + message = missed.body.decode() + assert f"no recorded interaction matches key post {CHAT_PATH} #" in message + assert f"closest recorded key is post {CHAT_PATH} #" in message + assert '"content": "x"' in message + assert '"content": "y"' in message + assert "re-record with E2E_FIXTURE_MODE=record" in message + + def test_query_params_are_part_of_the_identity(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "GET", "/openai/v1/models?purpose=batch") + assert provider.hits == ["GET /v1/models?purpose=batch"] + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + missed = call_edge(edge, "GET", "/openai/v1/models?purpose=other") + matched = call_edge(edge, "GET", "/openai/v1/models?purpose=batch") + assert missed.status_code == REPLAY_MISS_STATUS + assert matched.status_code == 200 + + def test_identical_requests_replay_their_responses_in_recorded_order(self, tmp_path: Path) -> None: + """A poll or retry loop repeats the same request and the proxy asserts + on the progression, so duplicates under one key stay FIFO.""" + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + first = json_object(call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")).body) + second = json_object(call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")).body) + assert first["hit"] == 1 + assert second["hit"] == 2 + + def test_exhausted_key_returns_the_miss_status(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + exhausted = call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert exhausted.status_code == REPLAY_MISS_STATUS + assert b"already consumed" in exhausted.body + + def test_non_json_bodies_match_by_canonical_digest_without_storing_them(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + opaque = b"custom_id one\ncustom_id two\n" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", "/openai/v1/files", body=opaque) + raw = this_tests_files(root)[0].read_text(encoding="utf-8") + interaction = Interaction.model_validate_json(raw) + assert interaction.request.body is None + assert interaction.request.file_sha256 is not None + assert interaction.request.file_bytes == len(opaque) + assert "custom_id" not in interaction.request.model_dump_json() + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + replayed = call_edge(edge, "POST", "/openai/v1/files", body=opaque) + assert replayed.status_code == 200 + + +class TestReplayLeftover: + def test_partially_consumed_recording_names_the_leftover(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + call_edge(edge, "GET", "/openai/v1/models") + source = replay_source(root) + with running_edge(ReplayEdge(source=source), REPLAY_MOUNTS) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + error = source.leftover_error(current_test_key()) + assert error is not None + assert "1 of 2 recorded interactions never consumed" in error + assert "e.g. get /openai/v1/models #" in error + assert "re-record with E2E_FIXTURE_MODE=record" in error + + def test_fully_consumed_recording_leaves_nothing(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + source = replay_source(root) + with running_edge(ReplayEdge(source=source), REPLAY_MOUNTS) as edge: + call_edge(edge, "POST", CHAT_PATH, body=chat_body("hi")) + assert source.leftover_error(current_test_key()) is None + + def test_test_without_recordings_has_no_leftover(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + assert isinstance(prepare_bundle(root), BundleRecorder) + assert replay_source(root).leftover_error("suite.py::test_never_recorded") is None + + def test_inert_outside_replay_mode(self, tmp_path: Path) -> None: + missing = tmp_path / "missing" + assert replay_leftover_error(mode_raw="", bundle_dir=missing, test_key="k") is None + assert replay_leftover_error(mode_raw="record", bundle_dir=missing, test_key="k") is None + + +class TestConcurrentReplay: + def test_parallel_identical_calls_serve_each_recording_exactly_once(self, tmp_path: Path) -> None: + """The edge server handles requests on concurrent threads and a burst + of parallel identical calls consumes one shared pool: no response + duplicated, none forgotten, nothing left over at teardown.""" + root = tmp_path / "bundle" + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + for ordinal in range(32): + recorder.record( + test_key=current_test_key(), + request=RecordedRequest(method="post", path=CHAT_PATH, headers={}, body={"n": "same"}), + response=RecordedHttpResponse( + status_code=200, + headers={"content-type": "application/json"}, + body_b64=base64.b64encode(json.dumps({"value": f"v{ordinal:02d}"}).encode()).decode(), + ), + ) + source = replay_source(root) + body = json.dumps({"n": "same"}).encode() + barrier = threading.Barrier(8) + with running_edge(ReplayEdge(source=source), REPLAY_MOUNTS) as edge: + + def consume(_: int) -> tuple[str, ...]: + barrier.wait() + return tuple( + str(json_object(call_edge(edge, "POST", CHAT_PATH, body=body).body)["value"]) + for _call in range(4) + ) + + with ThreadPoolExecutor(max_workers=8) as executor: + served = sorted(value for values in executor.map(consume, range(8)) for value in values) + assert served == [f"v{ordinal:02d}" for ordinal in range(32)] + assert source.leftover_error(current_test_key()) is None + + +class TestHandleEdgeRequestPure: + def test_unknown_mount_404s_naming_the_known_mounts(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + assert isinstance(prepare_bundle(root), BundleRecorder) + reply = handle_edge_request( + ReplayEdge(source=replay_source(root)), + {"openai": "https://api.openai.com", "anthropic": "https://api.anthropic.com"}, + "POST", + "/bedrock/model/invoke", + {}, + b"{}", + timeout=1.0, + ) + assert reply.status_code == 404 + assert b"unknown provider mount 'bedrock'" in reply.body + assert b"anthropic, openai" in reply.body + + def test_replay_serves_a_directly_recorded_interaction(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + recorder.record( + test_key=current_test_key(), + request=RecordedRequest(method="post", path=CHAT_PATH, headers={}, body={"prompt": "x"}), + response=RecordedHttpResponse( + status_code=201, headers={"x-upstream": "fake"}, body_b64=base64.b64encode(b"ok").decode() + ), + ) + reply = handle_edge_request( + ReplayEdge(source=replay_source(root)), + {"openai": "https://api.openai.com"}, + "POST", + CHAT_PATH, + {"authorization": "Bearer sk-anything"}, + json.dumps({"prompt": "x"}).encode(), + timeout=1.0, + ) + assert reply.status_code == 201 + assert reply.body == b"ok" + assert reply.headers == {"x-upstream": "fake"} + + +class TestApiBaseSeam: + def test_live_mode_returns_none(self, tmp_path: Path) -> None: + for mode_raw in ("live", ""): + assert ( + provider_edge_api_base( + "openai", + mode_raw=mode_raw, + bundle_dir=tmp_path / "bundle", + bind_host="127.0.0.1", + advertise_host="127.0.0.1", + ) + is None + ) + + def test_invalid_mode_raises_naming_the_value(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="cached"): + provider_edge_api_base( + "openai", + mode_raw="cached", + bundle_dir=tmp_path / "bundle", + bind_host="127.0.0.1", + advertise_host="127.0.0.1", + ) + + def test_unknown_mount_raises_naming_the_known_mounts(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="unknown provider mount 'bedrock'"): + provider_edge_api_base( + "bedrock", + mode_raw="record", + bundle_dir=tmp_path / "bundle", + bind_host="127.0.0.1", + advertise_host="127.0.0.1", + ) + + def test_record_mode_boots_one_shared_edge_and_prepares_the_bundle(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + first = provider_edge_api_base( + "openai", mode_raw="record", bundle_dir=root, bind_host="127.0.0.1", advertise_host="127.0.0.1" + ) + second = provider_edge_api_base( + "anthropic", mode_raw="record", bundle_dir=root, bind_host="127.0.0.1", advertise_host="127.0.0.1" + ) + assert first is not None and second is not None + assert first.endswith("/openai") + assert second.endswith("/anthropic") + assert first.rsplit("/", 1)[0] == second.rsplit("/", 1)[0] + assert (root / "manifest.json").is_file() From 0de829d3e44094a808e9c1166d12c4d86b29c6b0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:57:35 -0700 Subject: [PATCH 021/226] feat(cli): store the lite login credential in the OS keychain lite login used to write the minted cli-session key in cleartext to ~/.litellm/token.json. The secret material (key plus any JWT) now goes to the OS keychain through the optional keyring package, with the 0600 file kept for non-secret metadata and as the fallback on headless boxes. Legacy plaintext files keep authenticating and are migrated into the keychain, then scrubbed, on first read. A secret still on disk always outranks the keychain entry, so a failed keychain write can never resurrect a stale key. LITELLM_PROXY_API_KEY and --api-key precedence is unchanged, lite logout clears both stores and warns when the keychain will not release the entry, and ~/.litellm is created 0700 (tightened from 0755 where an older CLI left it broader). LITELLM_CLI_DISABLE_KEYRING=1 forces the file fallback. --- basedpyright-code-budget.json | 8 +- .../litellm_proxy_server/cli_token_usage.py | 2 +- litellm/litellm_core_utils/cli_keyring.py | 125 ++++ litellm/litellm_core_utils/cli_token_utils.py | 184 +++++- .../private_json.py | 10 + litellm/proxy/client/README.md | 12 +- litellm/proxy/client/cli/commands/agents.py | 4 +- litellm/proxy/client/cli/commands/auth.py | 148 +++-- .../client/cli/commands/claude_settings.py | 2 +- litellm/proxy/client/cli/commands/config.py | 6 +- litellm/proxy/client/cli/commands/up.py | 22 +- litellm/proxy/client/cli/main.py | 4 +- pyproject.toml | 2 + tests/test_litellm/conftest.py | 65 ++ .../test_cli_token_utils.py | 478 ++++++++++++--- .../proxy/client/cli/test_agents.py | 7 +- .../proxy/client/cli/test_auth_commands.py | 577 +++++++++--------- .../proxy/client/cli/test_claude_settings.py | 15 +- .../proxy/client/cli/test_config_commands.py | 4 +- .../proxy/client/cli/test_up_commands.py | 23 +- type-discipline-budget.json | 8 +- uv.lock | 100 ++- 22 files changed, 1295 insertions(+), 511 deletions(-) create mode 100644 litellm/litellm_core_utils/cli_keyring.py rename litellm/{proxy/client/cli/commands => litellm_core_utils}/private_json.py (64%) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 1ce71c5bd2c..59d56a3f63d 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5663 }, "reportMissingTypeArgument": { - "limit": 15557 + "limit": 15556 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 39043 + "limit": 39042 }, "reportUnknownParameterType": { - "limit": 19887 + "limit": 19886 }, "reportUnknownVariableType": { - "limit": 30574 + "limit": 30571 }, "reportUnnecessaryCast": { "limit": 117 diff --git a/cookbook/litellm_proxy_server/cli_token_usage.py b/cookbook/litellm_proxy_server/cli_token_usage.py index 6306970cdde..e6b3744019c 100644 --- a/cookbook/litellm_proxy_server/cli_token_usage.py +++ b/cookbook/litellm_proxy_server/cli_token_usage.py @@ -60,4 +60,4 @@ if __name__ == "__main__": print("\n💡 Tips:") print("1. Run 'litellm-proxy login' to authenticate first") print("2. Replace 'https://your-proxy.com' with your actual proxy URL") - print("3. The token is stored locally at ~/.litellm/token.json") + print("3. The token is stored in your OS keychain, or in ~/.litellm/token.json when there is none") diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py new file mode 100644 index 00000000000..873db64a728 --- /dev/null +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -0,0 +1,125 @@ +""" +CLI Keyring Access + +SDK-level access to the OS keychain (macOS Keychain, Windows Credential Manager, +Linux Secret Service) that holds the credential minted by `lite login`. + +The `keyring` package is optional and imported lazily, so importing this module +never pulls it in. Every failure is returned as a value: a machine with no +keychain, or one whose keychain is locked, must degrade to the token file rather +than break `lite` or the SDK. +""" + +import os +from dataclasses import dataclass +from typing import Final, Protocol, TypeAlias + +KEYRING_SERVICE: Final = "litellm-cli" +KEYRING_ACCOUNT: Final = "credential" +DISABLE_KEYRING_ENV_VAR: Final = "LITELLM_CLI_DISABLE_KEYRING" + +_DISABLED_VALUES: Final = frozenset(("1", "true", "yes", "on")) + + +@dataclass(frozen=True, slots=True) +class SecretFound: + blob: str + + +@dataclass(frozen=True, slots=True) +class SecretMissing: + pass + + +@dataclass(frozen=True, slots=True) +class SecretUnavailable: + pass + + +SecretRead: TypeAlias = SecretFound | SecretMissing | SecretUnavailable + + +class SecretVault(Protocol): + """The single slot holding the CLI credential's secret material.""" + + def read(self) -> SecretRead: ... + + def write(self, blob: str) -> bool: ... + + def erase(self) -> bool: ... + + +class KeyringApi(Protocol): + def get_password(self, service_name: str, username: str) -> str | None: ... + + def set_password(self, service_name: str, username: str, password: str) -> None: ... + + def delete_password(self, service_name: str, username: str) -> None: ... + + +def _keyring_disabled() -> bool: + return os.getenv(DISABLE_KEYRING_ENV_VAR, "").strip().lower() in _DISABLED_VALUES + + +def _import_keyring() -> KeyringApi | None: + try: + import keyring + except ImportError: + return None + return keyring + + +def _keyring_api() -> KeyringApi | None: + return None if _keyring_disabled() else _import_keyring() + + +@dataclass(frozen=True, slots=True) +class KeyringVault: + """The OS keychain, reached through the optional `keyring` package.""" + + def read(self) -> SecretRead: + api: Final = _keyring_api() + if api is None: + return SecretUnavailable() + try: + blob: Final = api.get_password(KEYRING_SERVICE, KEYRING_ACCOUNT) + except Exception: # noqa: BLE001 # backends raise outside keyring.errors; never break the SDK + return SecretUnavailable() + return SecretMissing() if blob is None else SecretFound(blob) + + def write(self, blob: str) -> bool: + api: Final = _keyring_api() + if api is None: + return False + try: + api.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, blob) + except Exception: # noqa: BLE001 # a keychain that refuses the write falls back to the token file + return False + return True + + def erase(self) -> bool: + if _import_keyring() is None: + return True + if _keyring_disabled(): + # a credential stored before the kill switch was set may still be in the keychain + return False + match self.read(): + case SecretUnavailable(): + return False + case SecretMissing(): + return True + case SecretFound(): + return self._delete() + + def _delete(self) -> bool: + api: Final = _keyring_api() + if api is None: + return False + try: + api.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT) + except Exception: # noqa: BLE001 # report the failure as a value so `lite logout` can warn + return False + return True + + +SYSTEM_KEYRING: Final[SecretVault] = KeyringVault() diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index a44ce431f4e..9960192180c 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -1,17 +1,68 @@ """ CLI Token Utilities -SDK-level utilities for reading CLI authentication tokens. +SDK-level utilities for reading the credential minted by `lite login`. + +Non-secret metadata lives in ~/.litellm/token.json. The secret material (the +bearer key, plus a JWT when one is issued) lives in the OS keychain when the +machine has one, and in that same 0600 file otherwise. This module hides the +split from callers, and migrates a legacy plaintext file into the keychain the +first time it reads one. + This module has no dependencies on proxy code and can be safely imported at the SDK level. """ -import json -import os +import contextlib import time -from collections.abc import Mapping from pathlib import Path +from types import MappingProxyType from typing import Final +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.litellm_core_utils.cli_keyring import ( + SYSTEM_KEYRING, + SecretFound, + SecretMissing, + SecretUnavailable, + SecretVault, +) +from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json + + +class CliTokenRecord(BaseModel): + """A stored CLI credential. + + `key is None` means the metadata was found but the secret could not be + produced: the keychain holds nothing for us, or we could not reach it. + """ + + model_config = ConfigDict(frozen=True, extra="allow") + + base_url: str = "" + key: str | None = None + user_id: str = "" + user_email: str = "" + user_role: str = "" + auth_header_name: str = "Authorization" + jwt_token: str = "" + timestamp: float = 0.0 + + +class CliTokenSecret(BaseModel): + """The secret material as stored in the OS keychain. + + `base_url` is duplicated from the metadata file purely as a pairing tag: a + secret minted for one server is never handed to another, even if the + metadata file is edited underneath us. + """ + + model_config = ConfigDict(frozen=True) + + base_url: str + key: str + jwt_token: str = "" + def get_cli_token_file_path() -> str: """Get the path to the CLI token file""" @@ -20,26 +71,39 @@ def get_cli_token_file_path() -> str: return str(config_dir / "token.json") -def load_cli_token() -> dict | None: - """Load CLI token data from file""" - token_file: Final = get_cli_token_file_path() - if not os.path.exists(token_file): +def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | None: + """Load the stored CLI credential, or None when this machine has none""" + record: Final = _read_token_file() + if record is None: return None + return _resolve_secret(record, vault) - try: - with open(token_file, "r") as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - return None + +def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> bool: + """Store a freshly minted credential. Returns whether the keychain took the secret""" + if record.key is None or not vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)): + _write_token_file(record) + return False + _write_token_file(_without_secret(record)) + return True + + +def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> bool: + """Remove the credential from both stores. Returns whether the keychain is now free of it""" + erased: Final = vault.erase() + Path(get_cli_token_file_path()).unlink(missing_ok=True) + return erased def get_litellm_gateway_api_key( expected_base_url: str | None = None, + *, + vault: SecretVault = SYSTEM_KEYRING, ) -> str | None: """ Get the stored CLI API key for use with LiteLLM SDK. - This function reads the token file created by `lite login` + This function reads the credential created by `lite login` and returns the API key for use in Python scripts. Args: @@ -47,6 +111,7 @@ def get_litellm_gateway_api_key( originally issued for this URL. Pass the target server URL to prevent credential leakage when the client is pointed at a different (possibly malicious) server. + vault: Where the secret material is stored. Defaults to the OS keychain. Returns: str: The API key if found (and origin matches), None otherwise @@ -62,25 +127,84 @@ def get_litellm_gateway_api_key( >>> base_url="https://your-proxy.com/v1" >>> ) """ - token_data: Final = load_cli_token() - if not token_data or "key" not in token_data: + record: Final = _read_token_file() + if record is None: return None - if expected_base_url is not None: - stored_url: Final = token_data.get("base_url") - if stored_url != expected_base_url.rstrip("/"): - return None - return token_data["key"] + if expected_base_url is not None and record.base_url != expected_base_url.rstrip("/"): + return None + resolved: Final = _resolve_secret(record, vault) + return None if resolved is None else resolved.key -def is_cli_token_fresh(token_data: Mapping[str, object], buffer_hours: float = 0.1) -> bool: - """Check whether a cached CLI token (as stored in token.json) is still - within its expiration window. Used by `lite auth print-token` to fail - fast, without a network round trip, once the cached token is past - `LITELLM_CLI_JWT_EXPIRATION_HOURS`.""" +def is_cli_token_fresh(token_data: CliTokenRecord, buffer_hours: float = 0.1) -> bool: + """Check whether a cached CLI token is still within its expiration window. + Used by `lite auth print-token` to fail fast, without a network round trip, + once the cached token is past `LITELLM_CLI_JWT_EXPIRATION_HOURS`.""" from litellm.constants import CLI_JWT_EXPIRATION_HOURS - timestamp: Final = token_data.get("timestamp") - if not isinstance(timestamp, (int, float)): - return False - age_hours: Final = (time.time() - timestamp) / 3600 + age_hours: Final = (time.time() - token_data.timestamp) / 3600 return age_hours < (CLI_JWT_EXPIRATION_HOURS - buffer_hours) + + +def _read_token_file() -> CliTokenRecord | None: + try: + raw: Final = Path(get_cli_token_file_path()).read_text() + except OSError: + return None + try: + return CliTokenRecord.model_validate_json(raw) + except ValidationError: + return None + + +def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: + match vault.read(): + case SecretFound(blob=blob): + return _apply_vault_secret(record, blob, vault) + case SecretMissing(): + return _migrate_file_secret(record, vault) + case SecretUnavailable(): + return record + + +def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) -> CliTokenRecord | None: + if record.key is not None: + # a secret still on disk means the last keychain write failed: the file outranks the vault + return _migrate_file_secret(record, vault) + try: + secret: Final = CliTokenSecret.model_validate_json(blob) + except ValidationError: + return _migrate_file_secret(record, vault) + if secret.base_url != record.base_url: + return _migrate_file_secret(record, vault) + _scrub_file_secret(record) + return record.model_copy(update=MappingProxyType({"key": secret.key, "jwt_token": secret.jwt_token})) + + +def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: + if record.key is None: + return None + if vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)): + _scrub_file_secret(record) + return record + + +def _scrub_file_secret(record: CliTokenRecord) -> None: + if record.key is None and not record.jwt_token: + return + with contextlib.suppress(OSError): + _write_token_file(_without_secret(record)) + + +def _without_secret(record: CliTokenRecord) -> CliTokenRecord: + return record.model_copy(update=MappingProxyType({"key": None, "jwt_token": ""})) + + +def _encode_secret(base_url: str, key: str, jwt_token: str) -> str: + return CliTokenSecret(base_url=base_url, key=key, jwt_token=jwt_token).model_dump_json() + + +def _write_token_file(record: CliTokenRecord) -> None: + path: Final = Path(get_cli_token_file_path()) + ensure_private_dir(path.parent) + write_private_json(str(path), record.model_dump(exclude_none=True)) diff --git a/litellm/proxy/client/cli/commands/private_json.py b/litellm/litellm_core_utils/private_json.py similarity index 64% rename from litellm/proxy/client/cli/commands/private_json.py rename to litellm/litellm_core_utils/private_json.py index 31062e4a799..32bc2e169e2 100644 --- a/litellm/proxy/client/cli/commands/private_json.py +++ b/litellm/litellm_core_utils/private_json.py @@ -1,10 +1,20 @@ import json import os +import stat import tempfile from collections.abc import Mapping from pathlib import Path from typing import Final +PRIVATE_DIR_MODE: Final = 0o700 + + +def ensure_private_dir(directory: Path) -> None: + """Create directory (and parents) owner-only, tightening it if it already exists group/world readable""" + directory.mkdir(mode=PRIVATE_DIR_MODE, parents=True, exist_ok=True) + if stat.S_IMODE(directory.stat().st_mode) & 0o077: + directory.chmod(PRIVATE_DIR_MODE) + def write_private_json(path: str, data: Mapping[str, object]) -> None: """Atomically write JSON to path with owner-only permissions (0600)""" diff --git a/litellm/proxy/client/README.md b/litellm/proxy/client/README.md index 6b28f43ac73..9ece4c2be3d 100644 --- a/litellm/proxy/client/README.md +++ b/litellm/proxy/client/README.md @@ -331,7 +331,7 @@ sequenceDiagram CLI->>Proxy: Poll /sso/cli/poll/login_id with poll_secret header Proxy->>CLI: Return {"status": "ready", "key": "jwt"} - CLI->>CLI: Save key to ~/.litellm/token.json + CLI->>CLI: Save key to the OS keychain (metadata to ~/.litellm/token.json) ``` ### Authentication Commands @@ -352,7 +352,7 @@ The CLI provides these authentication commands: 5. **Callback Processing**: SSO provider redirects back to proxy with state parameter 6. **User Code Verification**: Browser confirms the verification code shown in the CLI 7. **Polling**: CLI polls `/sso/cli/poll/{login_id}` with the polling secret header until the JWT is ready. When `CLI_SSO_CLAIM_MAP` is configured on the proxy, the poll response may include `attribution_metadata` (allowlisted scalar OIDC claims for client attribution). -8. **Token Storage**: CLI saves the authentication token to `~/.litellm/token.json` +8. **Token Storage**: CLI saves the key to the OS keychain and the non-secret session metadata to `~/.litellm/token.json` ### Benefits of This Approach @@ -364,11 +364,11 @@ The CLI provides these authentication commands: ### Token Storage -Authentication tokens are stored in `~/.litellm/token.json` with restricted file permissions (600). The stored token includes: +The key itself goes into the OS keychain (macOS Keychain, Windows Credential Manager, or the Linux Secret Service) under service `litellm-cli`, account `credential`. Only the non-secret session metadata is written to `~/.litellm/token.json`, in a `0700` directory with `0600` file permissions: ```json { - "key": "sk-...", + "base_url": "https://your-proxy.com", "user_id": "cli-user", "user_email": "user@example.com", "user_role": "cli", @@ -377,6 +377,10 @@ Authentication tokens are stored in `~/.litellm/token.json` with restricted file } ``` +Headless boxes and CI runners usually have no keychain. There the key stays in the same `0600` file alongside the metadata, exactly as it did before, and `lite login` tells you which of the two happened. Set `LITELLM_CLI_DISABLE_KEYRING=1` to force the file even where a keychain exists. A `token.json` written by an older `lite` keeps working and is moved into the keychain, and scrubbed from the file, the first time a keychain-capable `lite` reads it. + +`lite logout` clears both stores. If the keychain is locked at that moment it says so, and re-running it once the keychain is unlocked finishes the job. + The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while fresh and fails once it expires -- there is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`. ### Usage diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index ed2bf2be03d..e05e85ae483 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -8,7 +8,7 @@ from typing import Final import click import requests -from .auth import get_stored_api_key, login +from .auth import context_secret_vault, get_stored_api_key, login ANTHROPIC_BASE_URL_ENV: Final = "ANTHROPIC_BASE_URL" ANTHROPIC_AUTH_TOKEN_ENV: Final = "ANTHROPIC_AUTH_TOKEN" @@ -316,7 +316,7 @@ def resolve_api_key(ctx: click.Context) -> str: click.echo("No LiteLLM credentials found; starting login...") ctx.invoke(login) - api_key = get_stored_api_key(expected_base_url=base_url) + api_key = get_stored_api_key(expected_base_url=base_url, vault=context_secret_vault(ctx)) if not api_key: raise click.ClickException("Login did not produce an API key; cannot start the agent.") return api_key diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 0a0bcf80ee5..8b9ef5633da 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -1,9 +1,6 @@ -import json -import os import sys import time import webbrowser -from pathlib import Path from typing import Any, Final from urllib.parse import urlencode @@ -11,10 +8,19 @@ import click import requests from rich.console import Console from rich.table import Table -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import CLI_JWT_EXPIRATION_HOURS -from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh +from litellm.litellm_core_utils.cli_keyring import SYSTEM_KEYRING, SecretVault +from litellm.litellm_core_utils.cli_token_utils import ( + CliTokenRecord, + clear_cli_token, + get_cli_token_file_path, + get_litellm_gateway_api_key, + is_cli_token_fresh, + load_cli_token, + save_cli_token, +) from .claude_settings import ( CLAUDE_SETTINGS_PATH, @@ -22,18 +28,6 @@ from .claude_settings import ( ClaudeSettingsError, write_claude_settings, ) -from .private_json import write_private_json - - -class CliTokenData(TypedDict): - base_url: str - key: str - user_id: str - user_email: str - user_role: str - auth_header_name: str - jwt_token: str - timestamp: float class CliTeam(TypedDict, total=False): @@ -46,6 +40,7 @@ class CliTeam(TypedDict, total=False): class CliContextObj(TypedDict): base_url: str base_url_explicit: NotRequired[bool] + secret_vault: NotRequired[ReadOnly[SecretVault]] class CliPollData(TypedDict, total=False): @@ -76,50 +71,32 @@ class CliAuthResult(TypedDict): team_id: str | None +KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( + "Your credential is stored in your OS keychain, which could not be read. Unlock it and retry, or run 'lite login'." +) + + # Token storage utilities -def get_token_file_path() -> str: - """Get the path to store the authentication token""" - home_dir: Final = Path.home() - config_dir: Final = home_dir / ".litellm" - config_dir.mkdir(exist_ok=True) - return str(config_dir / "token.json") +def context_secret_vault(ctx: click.Context) -> SecretVault: + """Where this invocation reads and writes secret material; injectable through ctx.obj for tests""" + ctx_obj: Final[CliContextObj | None] = ctx.obj + if ctx_obj is None: + return SYSTEM_KEYRING + return ctx_obj.get("secret_vault") or SYSTEM_KEYRING -def save_token(token_data: CliTokenData) -> None: - """Save token data to file""" - write_private_json(get_token_file_path(), token_data) - - -def load_token() -> CliTokenData | None: - """Load token data from file""" - token_file: Final = get_token_file_path() - if not os.path.exists(token_file): - return None - - try: - with open(token_file, "r") as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - return None - - -def clear_token() -> None: - """Clear stored token""" - token_file: Final = get_token_file_path() - if os.path.exists(token_file): - os.remove(token_file) - - -def get_stored_api_key(expected_base_url: str | None = None) -> str | None: - """Get the stored API key from token file. +def get_stored_api_key( + expected_base_url: str | None = None, + *, + vault: SecretVault = SYSTEM_KEYRING, +) -> str | None: + """Get the stored API key. If expected_base_url is provided, the key is only returned when it was originally issued for that URL. This prevents credential leakage when the CLI is pointed at a different (possibly malicious) server. """ - from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key - - return get_litellm_gateway_api_key(expected_base_url=expected_base_url) + return get_litellm_gateway_api_key(expected_base_url=expected_base_url, vault=vault) # Team selection utilities @@ -689,23 +666,27 @@ def login(ctx: click.Context, config_claude: bool): api_key: Final = auth_result["api_key"] user_id: Final = auth_result["user_id"] - # Save token data. base_url is stored so we can verify origin - # before reusing the key on a subsequent CLI invocation. - save_token( - { - "base_url": base_url.rstrip("/"), - "key": api_key, - "user_id": user_id or "cli-user", - "user_email": "unknown", - "user_role": "cli", - "auth_header_name": "Authorization", - "jwt_token": "", - "timestamp": time.time(), - } + # base_url is stored so we can verify origin before reusing the + # key on a subsequent CLI invocation. + record: Final = CliTokenRecord( + base_url=base_url.rstrip("/"), + key=api_key, + user_id=user_id or "cli-user", + user_email="unknown", + user_role="cli", + auth_header_name="Authorization", + jwt_token="", + timestamp=time.time(), ) + in_keychain: Final = save_cli_token(record, vault=context_secret_vault(ctx)) click.echo("\nLogin successful!") click.echo(f"JWT Token: {api_key[:20]}...") + click.echo( + "Credential stored in your OS keychain." + if in_keychain + else f"No OS keychain available; credential stored in {get_cli_token_file_path()} (owner-only)." + ) click.echo("You can now use the CLI without specifying --api-key") if config_claude: @@ -736,10 +717,14 @@ def login(ctx: click.Context, config_claude: bool): @click.command(name="logout") -def logout(): +@click.pass_context +def logout(ctx: click.Context): """Logout and clear stored authentication""" - clear_token() - click.echo("Logged out successfully. Authentication token cleared.") + if clear_cli_token(vault=context_secret_vault(ctx)): + click.echo("Logged out successfully. Authentication token cleared.") + return + click.echo("Logged out. The local token file is gone, but the OS keychain entry could not be removed.") + click.echo("Unlock your keychain and run 'lite logout' again to clear it.") @click.command(name="print-token") @@ -753,7 +738,7 @@ def print_token(ctx: click.Context): expires after `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h); once expired, run `lite login` again. """ - token_data: Final = load_token() + token_data: Final = load_cli_token(vault=context_secret_vault(ctx)) if not token_data: click.echo("Not authenticated. Run 'lite login'.", err=True) sys.exit(1) @@ -765,7 +750,7 @@ def print_token(ctx: click.Context): ctx_obj: Final[CliContextObj] = ctx.obj if ctx_obj.get("base_url_explicit"): base_url: Final = ctx_obj["base_url"] - if token_data.get("base_url") != base_url.rstrip("/"): + if token_data.base_url != base_url.rstrip("/"): click.echo("Not authenticated for this server. Run 'lite login'.", err=True) sys.exit(1) @@ -773,33 +758,36 @@ def print_token(ctx: click.Context): click.echo("Token expired. Run 'lite login' again.", err=True) sys.exit(1) - api_key: Final = token_data.get("key") + api_key: Final = token_data.key if not api_key: - click.echo("No token available. Run 'lite login'.", err=True) + click.echo(KEYCHAIN_UNREACHABLE_MESSAGE, err=True) sys.exit(1) click.echo(api_key) @click.command(name="whoami") -def whoami(): +@click.pass_context +def whoami(ctx: click.Context): """Show current authentication status""" - token_data: Final = load_token() + token_data: Final = load_cli_token(vault=context_secret_vault(ctx)) if not token_data: click.echo("Not authenticated. Run 'lite login' to authenticate.") return click.echo("Authenticated") - click.echo(f"User Email: {token_data.get('user_email', 'Unknown')}") - click.echo(f"User ID: {token_data.get('user_id', 'Unknown')}") - click.echo(f"User Role: {token_data.get('user_role', 'Unknown')}") + click.echo(f"User Email: {token_data.user_email or 'Unknown'}") + click.echo(f"User ID: {token_data.user_id or 'Unknown'}") + click.echo(f"User Role: {token_data.user_role or 'Unknown'}") # Check if token is still valid (basic timestamp check) - timestamp: Final = token_data.get("timestamp", 0) - age_hours: Final = (time.time() - timestamp) / 3600 + age_hours: Final = (time.time() - token_data.timestamp) / 3600 click.echo(f"Token age: {age_hours:.1f} hours") + if token_data.key is None: + click.echo(KEYCHAIN_UNREACHABLE_MESSAGE) + if age_hours > CLI_JWT_EXPIRATION_HOURS: click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.") diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index e9a6a25a064..e18e5b1b7ee 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -15,7 +15,7 @@ from typing import Final from pydantic import JsonValue, TypeAdapter, ValidationError -from .private_json import write_private_json +from litellm.litellm_core_utils.private_json import write_private_json ENV_KEY: Final = "env" API_KEY_HELPER_KEY: Final = "apiKeyHelper" diff --git a/litellm/proxy/client/cli/commands/config.py b/litellm/proxy/client/cli/commands/config.py index 19dd407ba19..2715a0a9a38 100644 --- a/litellm/proxy/client/cli/commands/config.py +++ b/litellm/proxy/client/cli/commands/config.py @@ -10,7 +10,7 @@ from urllib.parse import urlparse import click from pydantic import TypeAdapter -from .private_json import write_private_json +from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json HIDDEN_COMMANDS_KEY: Final = "hidden_commands" @@ -42,7 +42,9 @@ def load_config() -> Mapping[str, str]: def save_config(config: Mapping[str, str]) -> None: """Save CLI config to file""" - write_private_json(get_config_file_path(), config) + config_file: Final = Path(get_config_file_path()) + ensure_private_dir(config_file.parent) + write_private_json(str(config_file), config) def get_config_value(key: str) -> str | None: diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index dd266b4afa1..a0fd4af8f72 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -14,10 +14,12 @@ from typing import IO, Final import click from pydantic import JsonValue, TypeAdapter, ValidationError -from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh +from litellm.litellm_core_utils.cli_keyring import SecretVault +from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh, load_cli_token +from litellm.litellm_core_utils.private_json import ensure_private_dir from .agents import AgentRunError, resolve_api_key, verify_proxy_key -from .auth import load_token, login +from .auth import context_secret_vault, login from .claude_settings import ( BACKUP_PATH, CLAUDE_SETTINGS_PATH, @@ -66,7 +68,7 @@ def secure_create(path: Path) -> Iterator[IO[str]]: def write_backup(record: BackupRecord, backup_path: Path | None = None) -> None: path: Final = backup_path if backup_path is not None else BACKUP_PATH - path.parent.mkdir(exist_ok=True) + ensure_private_dir(path.parent) with secure_create(path) as f: json.dump({"existed": record.existed, "content": record.content}, f, indent=2) @@ -103,10 +105,17 @@ def restore_claude_settings(settings_path: Path | None = None, backup_path: Path return record +def _has_fresh_login(base_url: str, vault: SecretVault) -> bool: + token_data: Final = load_cli_token(vault=vault) + if token_data is None or token_data.key is None or token_data.base_url != base_url: + return False + return is_cli_token_fresh(token_data) + + def _ensure_fresh_login(ctx: click.Context) -> None: base_url: Final = ctx.obj["base_url"].rstrip("/") - token_data = load_token() - if token_data and token_data.get("base_url") == base_url and is_cli_token_fresh(token_data): + vault: Final = context_secret_vault(ctx) + if _has_fresh_login(base_url, vault): return if not sys.stdin.isatty(): @@ -117,8 +126,7 @@ def _ensure_fresh_login(ctx: click.Context) -> None: click.echo("No fresh LiteLLM login found for this proxy; starting login...") ctx.invoke(login) - token_data = load_token() - if not token_data or token_data.get("base_url") != base_url or not is_cli_token_fresh(token_data): + if not _has_fresh_login(base_url, vault): raise UpError("Login did not produce a usable token; cannot start `lite up`.") diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 3a289736c66..664bf5a216c 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -9,7 +9,7 @@ from litellm._version import version as litellm_version from litellm.proxy.client.health import HealthManagementClient from .commands.agents import agent_commands -from .commands.auth import auth_group, get_stored_api_key, login, logout, whoami +from .commands.auth import auth_group, context_secret_vault, get_stored_api_key, login, logout, whoami from .commands.autoroute.commands import autoroute_group from .commands.chat import chat from .commands.config import config_commands, get_config_value, hidden_command_names @@ -94,7 +94,7 @@ def cli(ctx: click.Context, show_version: bool, base_url: str | None, api_key: s # If no API key provided via flag or environment variable, try to load from saved token. # Pass base_url so we only use the stored key when it was issued for this server. if api_key is None: - api_key = get_stored_api_key(expected_base_url=base_url) + api_key = get_stored_api_key(expected_base_url=base_url, vault=context_secret_vault(ctx)) ctx.obj["base_url"] = base_url ctx.obj["api_key"] = api_key diff --git a/pyproject.toml b/pyproject.toml index ffbc96eefb9..32921e14d31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ cli = [ "pyyaml>=6.0.3,<7.0", "requests>=2.32.0,<3.0", "InquirerPy>=0.3.4,<1.0", + "keyring>=25.6.0,<26.0", ] extra_proxy = [ "prisma>=0.11.0,<1.0", @@ -166,6 +167,7 @@ litellm-proxy = "litellm.proxy.client.cli:cli" dev = [ "diff-cover==9.7.2", "basedpyright==1.39.7", + "keyring==25.7.0", "pytest==9.0.3", "pytest-mock==3.15.1", "pytest-asyncio==1.3.0", diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index c0644c88291..ce0fd197538 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -22,6 +22,12 @@ import litellm from litellm import router as litellm_router_module from litellm import utils as litellm_utils_module from litellm._logging import ALL_LOGGERS +from litellm.litellm_core_utils.cli_keyring import ( + SecretFound, + SecretMissing, + SecretRead, + SecretUnavailable, +) from litellm.litellm_core_utils.prompt_templates import ( image_handling as image_handling_module, ) @@ -106,6 +112,65 @@ def isolate_host_proxy_base_url(monkeypatch): monkeypatch.delenv("PROXY_BASE_URL", raising=False) +@pytest.fixture(scope="function", autouse=True) +def isolate_host_os_keychain(monkeypatch): + """Keep any code path that resolves a CLI credential out of the developer's real OS keychain. + + Tests that exercise keychain behaviour inject their own vault instead. + """ + monkeypatch.setenv("LITELLM_CLI_DISABLE_KEYRING", "1") + + +class FakeSecretVault: + """In-memory stand-in for the OS keychain, injected wherever CLI credential storage is exercised. + + `available=False` models a keychain that is locked or has no backend, `writable=False` one that + refuses to store, and `erasable=False` one that will not release what it already holds. + """ + + def __init__( + self, + blob: str | None = None, + *, + available: bool = True, + writable: bool = True, + erasable: bool = True, + ) -> None: + self.blob: str | None = blob + self.available: bool = available + self.writable: bool = writable + self.erasable: bool = erasable + self.reads: int = 0 + self.writes: list[str] = [] + self.erases: int = 0 + + def read(self) -> SecretRead: + self.reads += 1 + if not self.available: + return SecretUnavailable() + return SecretMissing() if self.blob is None else SecretFound(self.blob) + + def write(self, blob: str) -> bool: + self.writes.append(blob) + if not (self.available and self.writable): + return False + self.blob = blob + return True + + def erase(self) -> bool: + self.erases += 1 + if not (self.available and self.erasable): + return False + self.blob = None + return True + + +@pytest.fixture +def secret_vault_factory(): + """Build FakeSecretVault instances; see its docstring for the failure modes it can model.""" + return FakeSecretVault + + def _run_coroutine_if_needed(result): if not asyncio.iscoroutine(result): return diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 27fc5eb4bd0..56ab6bcbfe0 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -1,89 +1,429 @@ -""" -Unit tests for CLI token utilities -""" - import json -import os -import tempfile -from pathlib import Path -from unittest.mock import mock_open, patch +import stat +import sys +import time import pytest -from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key +from litellm.constants import CLI_JWT_EXPIRATION_HOURS +from litellm.litellm_core_utils.cli_keyring import ( + DISABLE_KEYRING_ENV_VAR, + KEYRING_ACCOUNT, + KEYRING_SERVICE, + KeyringVault, + SecretFound, + SecretMissing, + SecretUnavailable, +) +from litellm.litellm_core_utils.cli_token_utils import ( + CliTokenRecord, + clear_cli_token, + get_cli_token_file_path, + get_litellm_gateway_api_key, + is_cli_token_fresh, + load_cli_token, + save_cli_token, +) + +SERVER = "https://proxy.example.com" +OTHER_SERVER = "https://other-proxy.example.com" -class TestCLITokenUtils: - """Test CLI token utility functions""" +@pytest.fixture +def isolated_home(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + return tmp_path - def test_get_litellm_gateway_api_key_success(self): - """Test getting CLI API key when token file exists and is valid""" - token_data = { - "key": "sk-test-cli-key-123", - "user_id": "test-user", - "user_email": "test@example.com", - "timestamp": 1234567890, - } - with ( - patch("os.path.exists", return_value=True), - patch("builtins.open", mock_open(read_data=json.dumps(token_data))), - patch( - "litellm.litellm_core_utils.cli_token_utils.get_cli_token_file_path", - return_value="/test/.litellm/token.json", - ), - ): +def _token_file(home): + return home / ".litellm" / "token.json" - result = get_litellm_gateway_api_key() - assert result == "sk-test-cli-key-123" +def _write_legacy_file(home, **overrides): + payload = { + "base_url": SERVER, + "key": "sk-legacy", + "user_id": "u-1", + "user_email": "user@example.com", + "user_role": "cli", + "timestamp": time.time(), + **overrides, + } + path = _token_file(home) + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(payload)) + path.chmod(0o600) + return path - def test_get_litellm_gateway_api_key_no_file(self): - """Test getting CLI API key when token file doesn't exist""" - with ( - patch("os.path.exists", return_value=False), - patch( - "litellm.litellm_core_utils.cli_token_utils.get_cli_token_file_path", - return_value="/test/.litellm/token.json", - ), - ): - result = get_litellm_gateway_api_key() +def _write_metadata_only_file(home): + """What a post-migration token.json looks like: everything except the secret material.""" + path = _token_file(home) + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps({"base_url": SERVER, "user_id": "u-1", "timestamp": time.time()})) + path.chmod(0o600) + return path - assert result is None - def test_get_litellm_gateway_api_key_invalid_json(self): - """Test getting CLI API key when token file has invalid JSON""" - with ( - patch("os.path.exists", return_value=True), - patch("builtins.open", mock_open(read_data="invalid json")), - patch( - "litellm.litellm_core_utils.cli_token_utils.get_cli_token_file_path", - return_value="/test/.litellm/token.json", - ), - ): +def _blob(base_url=SERVER, key="sk-vault", jwt_token=""): + return json.dumps({"base_url": base_url, "key": key, "jwt_token": jwt_token}) - result = get_litellm_gateway_api_key() - assert result is None +class TestGetCliTokenFilePath: + def test_points_at_the_home_config_file(self, isolated_home): + assert get_cli_token_file_path() == str(isolated_home / ".litellm" / "token.json") - def test_get_litellm_gateway_api_key_no_key_field(self): - """Test getting CLI API key when token file exists but has no key field""" - token_data = { - "user_id": "test-user", - "user_email": "test@example.com", - # Missing 'key' field - } + def test_does_not_create_the_directory(self, isolated_home): + """Merely asking for the path must not leave a directory behind, so an SDK import that + never logs in cannot create a ~/.litellm on someone's machine.""" + get_cli_token_file_path() - with ( - patch("os.path.exists", return_value=True), - patch("builtins.open", mock_open(read_data=json.dumps(token_data))), - patch( - "litellm.litellm_core_utils.cli_token_utils.get_cli_token_file_path", - return_value="/test/.litellm/token.json", - ), - ): + assert not (isolated_home / ".litellm").exists() - result = get_litellm_gateway_api_key() - assert result is None +class TestLoadCliToken: + def test_no_token_file_never_touches_the_keychain(self, isolated_home, secret_vault_factory): + """The SDK calls this on machines that never ran `lite login`; it must not prompt for + keychain access there.""" + vault = secret_vault_factory(blob=_blob()) + + assert load_cli_token(vault=vault) is None + assert vault.reads == 0 + + def test_secret_comes_from_the_vault_when_the_file_holds_only_metadata(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(blob=_blob(key="sk-from-keychain")) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-from-keychain" + assert "sk-from-keychain" not in _token_file(isolated_home).read_text() + + def test_jwt_token_round_trips_through_the_vault(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(blob=_blob(key="sk-a", jwt_token="jwt-a")) + + record = load_cli_token(vault=vault) + + assert (record.key, record.jwt_token) == ("sk-a", "jwt-a") + + def test_legacy_plaintext_file_still_authenticates_and_is_migrated(self, isolated_home, secret_vault_factory): + """A token.json written by an older `lite` keeps working, and reading it moves the secret + into the keychain and scrubs it from disk.""" + path = _write_legacy_file(isolated_home) + vault = secret_vault_factory() + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert json.loads(vault.blob)["key"] == "sk-legacy" + on_disk = json.loads(path.read_text()) + assert "key" not in on_disk + assert on_disk["user_email"] == "user@example.com" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + def test_legacy_file_survives_a_vault_that_refuses_to_store(self, isolated_home, secret_vault_factory): + """Scrubbing the only copy of the secret after a failed keychain write would log the user + out for good.""" + path = _write_legacy_file(isolated_home) + before = path.read_text() + + record = load_cli_token(vault=secret_vault_factory(writable=False)) + + assert record.key == "sk-legacy" + assert path.read_text() == before + + def test_a_secret_left_on_disk_outranks_a_stale_keychain_entry(self, isolated_home, secret_vault_factory): + """A failed keychain write leaves the fresh secret on disk while the vault still holds the + previous one; the next read must serve the file's secret and move it into the vault, never + resurrect the stale key or scrub the only copy of the fresh one.""" + path = _write_legacy_file(isolated_home, key="sk-fresh") + vault = secret_vault_factory(blob=_blob(key="sk-stale")) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-fresh" + assert json.loads(vault.blob)["key"] == "sk-fresh" + assert "key" not in json.loads(path.read_text()) + + def test_a_disk_secret_survives_when_the_stale_vault_refuses_the_rewrite( + self, isolated_home, secret_vault_factory + ): + path = _write_legacy_file(isolated_home, key="sk-fresh") + before = path.read_text() + + record = load_cli_token(vault=secret_vault_factory(blob=_blob(key="sk-stale"), writable=False)) + + assert record.key == "sk-fresh" + assert path.read_text() == before + + def test_legacy_file_survives_an_unreachable_vault_without_write_attempts( + self, isolated_home, secret_vault_factory + ): + path = _write_legacy_file(isolated_home) + before = path.read_text() + vault = secret_vault_factory(available=False) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert vault.writes == [] + assert path.read_text() == before + + def test_metadata_only_file_with_an_empty_vault_is_not_a_login(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + + assert load_cli_token(vault=secret_vault_factory()) is None + + def test_metadata_only_file_with_an_unreachable_vault_reports_a_missing_secret( + self, isolated_home, secret_vault_factory + ): + """The caller needs to tell "never logged in" apart from "locked keychain", so the record + comes back with no key rather than as None.""" + _write_metadata_only_file(isolated_home) + + record = load_cli_token(vault=secret_vault_factory(available=False)) + + assert record.key is None + assert record.user_id == "u-1" + + def test_a_secret_minted_for_another_server_is_never_handed_out(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + + assert load_cli_token(vault=secret_vault_factory(blob=_blob(base_url=OTHER_SERVER))) is None + + def test_a_secret_minted_for_another_server_loses_to_the_file(self, isolated_home, secret_vault_factory): + _write_legacy_file(isolated_home) + vault = secret_vault_factory(blob=_blob(base_url=OTHER_SERVER, key="sk-elsewhere")) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert json.loads(vault.blob)["key"] == "sk-legacy" + + def test_unreadable_vault_blob_falls_back_to_the_file_secret(self, isolated_home, secret_vault_factory): + _write_legacy_file(isolated_home) + + record = load_cli_token(vault=secret_vault_factory(blob="not json at all {{{")) + + assert record.key == "sk-legacy" + + def test_corrupt_token_file_is_not_a_login(self, isolated_home, secret_vault_factory): + _token_file(isolated_home).parent.mkdir() + _token_file(isolated_home).write_text("not json at all {{{") + + assert load_cli_token(vault=secret_vault_factory(blob=_blob())) is None + + +class TestGetLitellmGatewayApiKey: + def test_returns_the_vault_secret_when_the_origin_matches(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + + key = get_litellm_gateway_api_key(expected_base_url=SERVER, vault=secret_vault_factory(blob=_blob())) + + assert key == "sk-vault" + + def test_trailing_slash_on_the_expected_url_is_normalised(self, isolated_home, secret_vault_factory): + _write_metadata_only_file(isolated_home) + + key = get_litellm_gateway_api_key(expected_base_url=SERVER + "/", vault=secret_vault_factory(blob=_blob())) + + assert key == "sk-vault" + + def test_origin_mismatch_returns_nothing_without_reading_the_keychain(self, isolated_home, secret_vault_factory): + """Pointing the SDK at a different server must fail before the keychain is even consulted, + so a hostile base_url cannot provoke an unlock prompt.""" + _write_legacy_file(isolated_home) + vault = secret_vault_factory(blob=_blob()) + + assert get_litellm_gateway_api_key(expected_base_url=OTHER_SERVER, vault=vault) is None + assert vault.reads == 0 + + def test_no_token_file_returns_nothing(self, isolated_home, secret_vault_factory): + assert get_litellm_gateway_api_key(vault=secret_vault_factory(blob=_blob())) is None + + +class TestSaveCliToken: + def test_secret_goes_to_the_keychain_and_never_to_the_file(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + + stored = save_cli_token( + CliTokenRecord(base_url=SERVER, key="sk-new", user_id="u-1", timestamp=time.time()), + vault=vault, + ) + + assert stored is True + assert "sk-new" not in _token_file(isolated_home).read_text() + assert json.loads(vault.blob)["key"] == "sk-new" + assert load_cli_token(vault=vault).key == "sk-new" + + def test_falls_back_to_the_owner_only_file_when_there_is_no_keychain(self, isolated_home, secret_vault_factory): + stored = save_cli_token( + CliTokenRecord(base_url=SERVER, key="sk-new", timestamp=time.time()), + vault=secret_vault_factory(available=False), + ) + + path = _token_file(isolated_home) + assert stored is False + assert json.loads(path.read_text())["key"] == "sk-new" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert list(path.parent.glob(".tmp-*")) == [] + + def test_creates_the_config_directory_owner_only(self, isolated_home, secret_vault_factory): + """A 0755 ~/.litellm lets any local process list, and in the fallback case read, the + credential's directory.""" + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=secret_vault_factory()) + + assert stat.S_IMODE((isolated_home / ".litellm").stat().st_mode) == 0o700 + + def test_tightens_a_directory_left_group_readable_by_an_older_cli(self, isolated_home, secret_vault_factory): + config_dir = isolated_home / ".litellm" + config_dir.mkdir(mode=0o755) + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=secret_vault_factory()) + + assert stat.S_IMODE(config_dir.stat().st_mode) == 0o700 + + def test_a_failed_write_leaves_the_previous_credential_intact(self, isolated_home, secret_vault_factory, monkeypatch): + path = _write_legacy_file(isolated_home) + before = path.read_text() + + def _explode(*args, **kwargs): + raise TypeError("not serialisable") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + + with pytest.raises(TypeError): + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=secret_vault_factory(available=False)) + + assert path.read_text() == before + assert list(path.parent.glob(".tmp-*")) == [] + + +class TestClearCliToken: + def test_removes_the_credential_from_both_stores(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) + + assert clear_cli_token(vault=vault) is True + assert vault.blob is None + assert not _token_file(isolated_home).exists() + assert load_cli_token(vault=vault) is None + + def test_reports_a_keychain_that_will_not_release_the_secret(self, isolated_home, secret_vault_factory): + _write_legacy_file(isolated_home) + vault = secret_vault_factory(blob=_blob(), erasable=False) + + assert clear_cli_token(vault=vault) is False + assert not _token_file(isolated_home).exists() + + def test_is_safe_when_nothing_was_ever_stored(self, isolated_home, secret_vault_factory): + assert clear_cli_token(vault=secret_vault_factory()) is True + + +class TestIsCliTokenFresh: + def test_a_just_issued_token_is_fresh(self): + assert is_cli_token_fresh(CliTokenRecord(timestamp=time.time())) is True + + def test_a_token_past_its_expiry_is_stale(self): + stale = CliTokenRecord(timestamp=time.time() - (CLI_JWT_EXPIRATION_HOURS + 1) * 3600) + + assert is_cli_token_fresh(stale) is False + + def test_the_buffer_retires_a_token_just_before_it_expires(self): + almost = CliTokenRecord(timestamp=time.time() - (CLI_JWT_EXPIRATION_HOURS * 3600 - 60)) + + assert is_cli_token_fresh(almost, buffer_hours=0.1) is False + + +class _FakeKeyringModule: + def __init__(self, stored=None, *, get_error=None, set_error=None, delete_error=None): + self.stored = stored + self.get_error = get_error + self.set_error = set_error + self.delete_error = delete_error + self.calls = [] + + def get_password(self, service_name, username): + self.calls.append(("get", service_name, username)) + if self.get_error is not None: + raise self.get_error + return self.stored + + def set_password(self, service_name, username, password): + self.calls.append(("set", service_name, username)) + if self.set_error is not None: + raise self.set_error + self.stored = password + + def delete_password(self, service_name, username): + self.calls.append(("delete", service_name, username)) + if self.delete_error is not None: + raise self.delete_error + self.stored = None + + +@pytest.fixture +def install_fake_keyring(monkeypatch): + def _install(fake): + monkeypatch.delenv(DISABLE_KEYRING_ENV_VAR, raising=False) + monkeypatch.setitem(sys.modules, "keyring", fake) + return fake + + return _install + + +class TestKeyringVault: + def test_round_trips_through_the_installed_keyring(self, install_fake_keyring): + fake = install_fake_keyring(_FakeKeyringModule()) + vault = KeyringVault() + + assert vault.write("blob-1") is True + assert vault.read() == SecretFound("blob-1") + assert vault.erase() is True + assert vault.read() == SecretMissing() + assert {call[1:] for call in fake.calls} == {(KEYRING_SERVICE, KEYRING_ACCOUNT)} + + def test_the_kill_switch_reports_no_keychain(self, monkeypatch): + """`LITELLM_CLI_DISABLE_KEYRING` has to work without importing keyring, because keyring + caches its backend on first use and cannot be reconfigured later. Erase still fails: a + credential stored before the switch was set may be in the keychain, and with reads + disabled `lite logout` cannot verify it is gone, so it must warn instead.""" + monkeypatch.setenv(DISABLE_KEYRING_ENV_VAR, "1") + vault = KeyringVault() + + assert vault.read() == SecretUnavailable() + assert vault.write("blob-1") is False + assert vault.erase() is False + + def test_an_uninstalled_keyring_library_degrades_to_the_file(self, monkeypatch): + """keyring is an optional extra, so the SDK must survive its absence rather than raise on + the hot path.""" + monkeypatch.delenv(DISABLE_KEYRING_ENV_VAR, raising=False) + monkeypatch.setitem(sys.modules, "keyring", None) + vault = KeyringVault() + + assert vault.read() == SecretUnavailable() + assert vault.write("blob-1") is False + assert vault.erase() is True + + def test_a_locked_keychain_is_reported_not_raised(self, install_fake_keyring): + install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("keyring is locked"))) + + assert KeyringVault().read() == SecretUnavailable() + + def test_a_refused_write_is_reported_not_raised(self, install_fake_keyring): + install_fake_keyring(_FakeKeyringModule(set_error=RuntimeError("no backend"))) + + assert KeyringVault().write("blob-1") is False + + def test_a_refused_delete_is_reported_so_logout_can_warn(self, install_fake_keyring): + install_fake_keyring(_FakeKeyringModule(stored="blob-1", delete_error=RuntimeError("locked"))) + + assert KeyringVault().erase() is False + + def test_erasing_a_locked_keychain_is_a_failure(self, install_fake_keyring): + install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("locked"))) + + assert KeyringVault().erase() is False diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py index a23c573047f..c2858c84c6d 100644 --- a/tests/test_litellm/proxy/client/cli/test_agents.py +++ b/tests/test_litellm/proxy/client/cli/test_agents.py @@ -672,8 +672,9 @@ class TestAgentCommands: assert "LITELLM_PROXY_API_KEY" in result.output mock_run.assert_not_called() - def test_interactive_without_key_logs_in_then_launches(self): + def test_interactive_without_key_logs_in_then_launches(self, secret_vault_factory): captured = {} + vault = secret_vault_factory() @click.command() def fake_login(): @@ -695,12 +696,12 @@ class TestAgentCommands: result = self.runner.invoke( _agent_command("claude"), [], - obj={"base_url": "http://localhost:4000", "api_key": None}, + obj={"base_url": "http://localhost:4000", "api_key": None, "secret_vault": vault}, ) assert result.exit_code == 0, result.output assert captured["api_key"] == "sk-after-login" - mock_get.assert_called_once_with(expected_base_url="http://localhost:4000") + mock_get.assert_called_once_with(expected_base_url="http://localhost:4000", vault=vault) def test_child_exit_code_reaches_the_shell(self): with patch(f"{AGENTS_MODULE}.run_agent", side_effect=SystemExit(42)): diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 59048067674..e93f05cb4aa 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -4,7 +4,7 @@ import stat import sys import time from pathlib import Path -from unittest.mock import Mock, mock_open, patch +from unittest.mock import Mock, patch sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path @@ -13,21 +13,38 @@ import pytest from click.testing import CliRunner from litellm.constants import CLI_JWT_EXPIRATION_HOURS +from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord, save_cli_token from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.auth import ( - clear_token, + KEYCHAIN_UNREACHABLE_MESSAGE, get_stored_api_key, - get_token_file_path, - load_token, login, logout, print_token, - save_token, whoami, ) from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner +@pytest.fixture +def isolated_home(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.delenv("LITELLM_PROXY_URL", raising=False) + monkeypatch.delenv("LITELLM_PROXY_API_KEY", raising=False) + return tmp_path + + +def _write_home_json(home: Path, filename: str, payload: dict[str, object]) -> None: + litellm_dir = home / ".litellm" + litellm_dir.mkdir(exist_ok=True) + (litellm_dir / filename).write_text(json.dumps(payload)) + + +def _secret_blob(base_url: str, key: str) -> str: + return json.dumps({"base_url": base_url, "key": key, "jwt_token": ""}) + + def _mock_cli_sso_start_response( login_id: str = "cli-session-uuid-456", poll_secret: str = "poll-secret", @@ -176,200 +193,50 @@ class TestStartCliSsoFlowErrors: assert "https://unreachable.example.com/sso/cli/start" in message -class TestTokenUtilities: - """Test token file utility functions""" +class TestStoredApiKeyLookup: + """`get_stored_api_key` is what every other `lite` subcommand authenticates with, so the + keychain split and the origin check both have to be invisible to it.""" - def test_get_token_file_path(self): - """Test getting token file path""" - with ( - patch("pathlib.Path.home") as mock_home, - patch("pathlib.Path.mkdir") as mock_mkdir, - ): - mock_home.return_value = Path("/home/user") + def test_returns_the_secret_the_keychain_holds(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "user_id": "u-1"}) + vault = secret_vault_factory(blob=_secret_blob("https://real-proxy.com", "sk-from-keychain")) - result = get_token_file_path() + assert get_stored_api_key(vault=vault) == "sk-from-keychain" - assert result == "/home/user/.litellm/token.json" - mock_mkdir.assert_called_once_with(exist_ok=True) + def test_returns_a_legacy_plaintext_key(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "key": "sk-legacy"}) - def test_get_token_file_path_creates_directory(self): - """Test that get_token_file_path creates the config directory""" - with ( - patch("pathlib.Path.home") as mock_home, - patch("pathlib.Path.mkdir") as mock_mkdir, - ): - mock_home.return_value = Path("/home/user") + assert get_stored_api_key(vault=secret_vault_factory()) == "sk-legacy" - get_token_file_path() + def test_no_token_at_all_returns_nothing(self, isolated_home, secret_vault_factory): + assert get_stored_api_key(vault=secret_vault_factory()) is None - mock_mkdir.assert_called_once_with(exist_ok=True) + def test_metadata_without_a_secret_returns_nothing(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "user_id": "u-1"}) - def test_save_token(self, tmp_path): - """Test saving token data to file""" - token_data = { - "key": "test-key", - "user_id": "test-user", - "timestamp": 1234567890, - } - token_file = tmp_path / "token.json" + assert get_stored_api_key(vault=secret_vault_factory()) is None - with patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path: - mock_path.return_value = str(token_file) + def test_matching_base_url_returns_the_key(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "key": "sk-prod"}) - save_token(token_data) + assert get_stored_api_key("https://real-proxy.com", vault=secret_vault_factory()) == "sk-prod" - assert json.loads(token_file.read_text()) == token_data - assert stat.S_IMODE(token_file.stat().st_mode) == 0o600 + def test_trailing_slash_on_the_expected_url_is_normalised(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "key": "sk-prod"}) - def test_load_token_success(self): - """Test loading token data from file successfully""" - token_data = { - "key": "test-key", - "user_id": "test-user", - "timestamp": 1234567890, - } + assert get_stored_api_key("https://real-proxy.com/", vault=secret_vault_factory()) == "sk-prod" - with ( - patch("builtins.open", mock_open(read_data=json.dumps(token_data))), - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=True), - ): - mock_path.return_value = "/test/path/token.json" + def test_mismatched_base_url_withholds_the_key(self, isolated_home, secret_vault_factory): + _write_home_json(isolated_home, "token.json", {"base_url": "https://real-proxy.com", "key": "sk-prod"}) - result = load_token() + assert get_stored_api_key("https://evil.com", vault=secret_vault_factory()) is None - assert result == token_data + def test_old_tokens_without_a_base_url_are_rejected_when_an_origin_is_expected( + self, isolated_home, secret_vault_factory + ): + _write_home_json(isolated_home, "token.json", {"key": "sk-old-token"}) - def test_load_token_file_not_exists(self): - """Test loading token when file doesn't exist""" - with ( - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=False), - ): - mock_path.return_value = "/test/path/token.json" - - result = load_token() - - assert result is None - - def test_load_token_json_decode_error(self): - """Test loading token with invalid JSON""" - with ( - patch("builtins.open", mock_open(read_data="invalid json")), - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=True), - ): - mock_path.return_value = "/test/path/token.json" - - result = load_token() - - assert result is None - - def test_load_token_io_error(self): - """Test loading token with IO error""" - with ( - patch("builtins.open", side_effect=OSError("Permission denied")), - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=True), - ): - mock_path.return_value = "/test/path/token.json" - - result = load_token() - - assert result is None - - def test_clear_token_file_exists(self): - """Test clearing token when file exists""" - with ( - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=True), - patch("os.remove") as mock_remove, - ): - mock_path.return_value = "/test/path/token.json" - - clear_token() - - mock_remove.assert_called_once_with("/test/path/token.json") - - def test_clear_token_file_not_exists(self): - """Test clearing token when file doesn't exist""" - with ( - patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, - patch("os.path.exists", return_value=False), - patch("os.remove") as mock_remove, - ): - mock_path.return_value = "/test/path/token.json" - - clear_token() - - mock_remove.assert_not_called() - - def test_get_stored_api_key_success(self): - """Test getting stored API key successfully""" - token_data = {"key": "test-api-key-123", "user_id": "test-user"} - - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - result = get_stored_api_key() - assert result == "test-api-key-123" - - def test_get_stored_api_key_no_token(self): - """Test getting stored API key when no token exists""" - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=None, - ): - result = get_stored_api_key() - assert result is None - - def test_get_stored_api_key_no_key_field(self): - """Test getting stored API key when token has no key field""" - token_data = {"user_id": "test-user"} - - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - result = get_stored_api_key() - assert result is None - - def test_get_stored_api_key_base_url_match(self): - """Stored key is returned when expected_base_url matches stored origin""" - token_data = {"key": "sk-prod", "base_url": "https://real-proxy.com"} - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - assert get_stored_api_key(expected_base_url="https://real-proxy.com") == "sk-prod" - - def test_get_stored_api_key_base_url_match_trailing_slash(self): - """Trailing slash on expected_base_url is normalised before comparison""" - token_data = {"key": "sk-prod", "base_url": "https://real-proxy.com"} - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - assert get_stored_api_key(expected_base_url="https://real-proxy.com/") == "sk-prod" - - def test_get_stored_api_key_base_url_mismatch(self): - """Stored key is NOT returned when expected_base_url differs from stored origin""" - token_data = {"key": "sk-prod", "base_url": "https://real-proxy.com"} - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - assert get_stored_api_key(expected_base_url="https://evil.com") is None - - def test_get_stored_api_key_old_token_no_base_url(self): - """Old tokens without a base_url field are rejected when origin check is requested""" - token_data = {"key": "sk-old-token"} - with patch( - "litellm.litellm_core_utils.cli_token_utils.load_cli_token", - return_value=token_data, - ): - assert get_stored_api_key(expected_base_url="https://real-proxy.com") is None + assert get_stored_api_key("https://real-proxy.com", vault=secret_vault_factory()) is None class TestLoginCommand: @@ -402,7 +269,7 @@ class TestLoginCommand: return_value=_mock_cli_sso_start_response(login_id="cli-test-uuid-123"), ) as mock_post, patch("requests.get", return_value=mock_response) as mock_get, - patch("litellm.proxy.client.cli.commands.auth.save_token") as mock_save, + patch("litellm.proxy.client.cli.commands.auth.save_cli_token") as mock_save, patch("litellm.proxy.client.cli.interface.show_commands") as mock_show_commands, ): result = self.runner.invoke(login, obj=mock_context.obj) @@ -424,8 +291,8 @@ class TestLoginCommand: # Verify JWT was saved mock_save.assert_called_once() saved_data = mock_save.call_args[0][0] - assert saved_data["key"] == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.jwt" - assert saved_data["user_id"] == "test-user-123" + assert saved_data.key == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.jwt" + assert saved_data.user_id == "test-user-123" # Verify commands were shown mock_show_commands.assert_called_once() @@ -557,7 +424,7 @@ class TestLogoutCommand: def test_logout_success(self): """Test successful logout""" - with patch("litellm.proxy.client.cli.commands.auth.clear_token") as mock_clear: + with patch("litellm.proxy.client.cli.commands.auth.clear_cli_token") as mock_clear: result = self.runner.invoke(logout) assert result.exit_code == 0 @@ -574,14 +441,15 @@ class TestWhoamiCommand: def test_whoami_authenticated(self): """Test whoami when user is authenticated""" - token_data = { - "user_email": "test@example.com", - "user_id": "test-user-123", - "user_role": "admin", - "timestamp": time.time() - 3600, # 1 hour ago - } + token_data = CliTokenRecord( + user_email="test@example.com", + user_id="test-user-123", + user_role="admin", + key="sk-live", + timestamp=time.time() - 3600, + ) - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=token_data): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=token_data): result = self.runner.invoke(whoami) assert result.exit_code == 0 @@ -593,7 +461,7 @@ class TestWhoamiCommand: def test_whoami_not_authenticated(self): """Test whoami when user is not authenticated""" - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=None): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=None): result = self.runner.invoke(whoami) assert result.exit_code == 0 @@ -602,14 +470,15 @@ class TestWhoamiCommand: def test_whoami_old_token(self): """Test whoami with old token showing warning""" - token_data = { - "user_email": "test@example.com", - "user_id": "test-user-123", - "user_role": "admin", - "timestamp": time.time() - (25 * 3600), # 25 hours ago - } + token_data = CliTokenRecord( + user_email="test@example.com", + user_id="test-user-123", + user_role="admin", + key="sk-live", + timestamp=time.time() - (25 * 3600), + ) - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=token_data): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=token_data): result = self.runner.invoke(whoami) assert result.exit_code == 0 @@ -618,12 +487,9 @@ class TestWhoamiCommand: def test_whoami_missing_fields(self): """Test whoami with token missing some fields""" - token_data = { - "timestamp": time.time() - 3600 - # Missing user_email, user_id, user_role - } + token_data = CliTokenRecord(key="sk-live", timestamp=time.time() - 3600) - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=token_data): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=token_data): result = self.runner.invoke(whoami) assert result.exit_code == 0 @@ -632,16 +498,16 @@ class TestWhoamiCommand: def test_whoami_no_timestamp(self): """Test whoami with token missing timestamp""" - token_data = { - "user_email": "test@example.com", - "user_id": "test-user-123", - "user_role": "admin", - # Missing timestamp - } + token_data = CliTokenRecord( + user_email="test@example.com", + user_id="test-user-123", + user_role="admin", + key="sk-live", + ) with ( patch( - "litellm.proxy.client.cli.commands.auth.load_token", + "litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=token_data, ), patch("time.time", return_value=1000), @@ -701,7 +567,7 @@ class TestCLIKeyRegenerationFlow: return_value=_mock_cli_sso_start_response(login_id="cli-session-uuid-456"), ), patch("requests.get", side_effect=[mock_first_response, mock_second_response]) as mock_get, - patch("litellm.proxy.client.cli.commands.auth.save_token") as mock_save, + patch("litellm.proxy.client.cli.commands.auth.save_cli_token") as mock_save, patch("litellm.proxy.client.cli.interface.show_commands") as mock_show_commands, patch("click.prompt", return_value="2"), ): # User selects index 2 @@ -734,8 +600,8 @@ class TestCLIKeyRegenerationFlow: # Verify JWT was saved mock_save.assert_called_once() saved_data = mock_save.call_args[0][0] - assert saved_data["key"] == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.team-beta.jwt" - assert saved_data["user_id"] == "test-user-456" + assert saved_data.key == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.team-beta.jwt" + assert saved_data.user_id == "test-user-456" mock_show_commands.assert_called_once() @@ -762,7 +628,7 @@ class TestCLIKeyRegenerationFlow: return_value=_mock_cli_sso_start_response(login_id="cli-session-uuid-solo"), ), patch("requests.get", return_value=mock_response), - patch("litellm.proxy.client.cli.commands.auth.save_token") as mock_save, + patch("litellm.proxy.client.cli.commands.auth.save_cli_token") as mock_save, patch("litellm.proxy.client.cli.interface.show_commands"), ): result = self.runner.invoke(login, obj=mock_context.obj) @@ -780,8 +646,8 @@ class TestCLIKeyRegenerationFlow: # Verify JWT was saved mock_save.assert_called_once() saved_data = mock_save.call_args[0][0] - assert saved_data["key"] == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.no-team.jwt" - assert saved_data["user_id"] == "test-user-solo" + assert saved_data.key == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.no-team.jwt" + assert saved_data.user_id == "test-user-solo" class TestPrintTokenCommand: @@ -810,7 +676,7 @@ class TestPrintTokenCommand: self.runner = CliRunner() def test_no_stored_token_fails_cleanly(self): - with patch("litellm.proxy.client.cli.commands.auth.load_token", return_value=None): + with patch("litellm.proxy.client.cli.commands.auth.load_cli_token", return_value=None): result = self.runner.invoke(print_token, obj={}) assert result.exit_code != 0 @@ -822,12 +688,12 @@ class TestPrintTokenCommand: one). Must use token.json's own base_url, not a hardcoded default.""" with ( patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "https://litellm-proxy.corp.com", - "key": "sk-prod-fresh", - "timestamp": time.time(), - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="https://litellm-proxy.corp.com", + key="sk-prod-fresh", + timestamp=time.time(), + ), ), patch("requests.post") as mock_post, ): @@ -844,12 +710,12 @@ class TestPrintTokenCommand: token minted for proxy A must not reach a helper invocation aimed at proxy B, even though the token itself is otherwise fresh.""" with patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "https://other-server.com", - "key": "sk-should-not-print", - "timestamp": time.time(), - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="https://other-server.com", + key="sk-should-not-print", + timestamp=time.time(), + ), ): result = self.runner.invoke( print_token, @@ -863,12 +729,12 @@ class TestPrintTokenCommand: """`lite up`'s own bound invocation shape: --base-url matching the token's origin must succeed exactly like the bare/legacy invocation does.""" with patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "http://localhost:4000", - "key": "sk-matches", - "timestamp": time.time(), - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="http://localhost:4000", + key="sk-matches", + timestamp=time.time(), + ), ): result = self.runner.invoke( print_token, @@ -884,12 +750,12 @@ class TestPrintTokenCommand: frequently).""" with ( patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "http://localhost:4000", - "key": "sk-cached-fresh", - "timestamp": time.time(), - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="http://localhost:4000", + key="sk-cached-fresh", + timestamp=time.time(), + ), ), patch("requests.post") as mock_post, ): @@ -908,12 +774,12 @@ class TestPrintTokenCommand: with ( patch( - "litellm.proxy.client.cli.commands.auth.load_token", - return_value={ - "base_url": "http://localhost:4000", - "key": "sk-stale-key", - "timestamp": old_timestamp, - }, + "litellm.proxy.client.cli.commands.auth.load_cli_token", + return_value=CliTokenRecord( + base_url="http://localhost:4000", + key="sk-stale-key", + timestamp=old_timestamp, + ), ), patch("requests.post") as mock_post, ): @@ -925,25 +791,11 @@ class TestPrintTokenCommand: mock_post.assert_not_called() -def _write_home_json(home: Path, filename: str, payload: dict[str, object]) -> None: - litellm_dir = home / ".litellm" - litellm_dir.mkdir(exist_ok=True) - (litellm_dir / filename).write_text(json.dumps(payload)) - - class TestPrintTokenWithConfigFile: """A config-file base_url is a drop-in replacement for exporting LITELLM_PROXY_URL, so print-token must treat it as an explicit server choice: a token minted for a different proxy is never handed out.""" - @pytest.fixture - def isolated_home(self, monkeypatch, tmp_path): - monkeypatch.setenv("HOME", str(tmp_path)) - monkeypatch.setenv("USERPROFILE", str(tmp_path)) - monkeypatch.delenv("LITELLM_PROXY_URL", raising=False) - monkeypatch.delenv("LITELLM_PROXY_API_KEY", raising=False) - return tmp_path - def test_config_base_url_mismatch_fails_closed(self, isolated_home): _write_home_json( isolated_home, @@ -1001,37 +853,196 @@ class TestPrintTokenWithConfigFile: assert result.stdout.strip() == "sk-issued-for-a" -class TestSaveTokenPrivateWrite: - """token.json holds the real API key: it must never be world-readable at any - instant, and a failed write must not destroy the previously stored token.""" +class TestFileFallbackStorage: + """On a headless box with no keychain the token file is still the only store, so it has to + stay owner-only and survive a failed write.""" - @pytest.fixture - def isolated_home(self, monkeypatch, tmp_path): - monkeypatch.setenv("HOME", str(tmp_path)) - monkeypatch.setenv("USERPROFILE", str(tmp_path)) - monkeypatch.delenv("LITELLM_PROXY_URL", raising=False) - monkeypatch.delenv("LITELLM_PROXY_API_KEY", raising=False) - return tmp_path - - def test_save_token_owner_only_permissions_and_no_temp_leftovers(self, isolated_home): - save_token({"key": "sk-secret", "user_id": "u-1", "timestamp": 1234567890}) + def test_owner_only_file_and_directory_with_no_temp_leftovers(self, isolated_home, secret_vault_factory): + save_cli_token( + CliTokenRecord(base_url="https://proxy.example.com", key="sk-secret", user_id="u-1", timestamp=1234567890), + vault=secret_vault_factory(available=False), + ) token_file = isolated_home / ".litellm" / "token.json" - assert json.loads(token_file.read_text()) == {"key": "sk-secret", "user_id": "u-1", "timestamp": 1234567890} + assert json.loads(token_file.read_text())["key"] == "sk-secret" assert stat.S_IMODE(token_file.stat().st_mode) == 0o600 + assert stat.S_IMODE(token_file.parent.stat().st_mode) == 0o700 assert list(token_file.parent.glob(".tmp-*")) == [] - def test_save_token_failure_mid_write_preserves_existing_token(self, isolated_home): + def test_a_failed_write_preserves_the_existing_token(self, isolated_home, secret_vault_factory, monkeypatch): _write_home_json(isolated_home, "token.json", {"key": "sk-original", "timestamp": 1234567890}) token_file = isolated_home / ".litellm" / "token.json" + def _explode(*args, **kwargs): + raise TypeError("not serialisable") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + with pytest.raises(TypeError): - save_token({"key": object()}) + save_cli_token(CliTokenRecord(key="sk-new"), vault=secret_vault_factory(available=False)) assert json.loads(token_file.read_text()) == {"key": "sk-original", "timestamp": 1234567890} assert list(token_file.parent.glob(".tmp-*")) == [] +class TestKeychainBackedCommands: + """End-to-end through the `lite` commands: the secret lives in the keychain, the file keeps + only metadata, and every command still reads and writes through that split.""" + + def setup_method(self): + self.runner = CliRunner() + + def _login(self, vault, base_url="https://test.example.com"): + poll_response = Mock() + poll_response.status_code = 200 + poll_response.json.return_value = { + "status": "ready", + "key": "sk-minted", + "user_id": "test-user-123", + "team_id": "team-1", + "teams": ["team-1"], + } + with ( + patch("webbrowser.open"), + patch("requests.post", return_value=_mock_cli_sso_start_response()), + patch("requests.get", return_value=poll_response), + patch("litellm.proxy.client.cli.interface.show_commands"), + ): + return self.runner.invoke(login, obj={"base_url": base_url, "secret_vault": vault}) + + def test_login_puts_the_secret_in_the_keychain_and_not_in_the_file(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + + result = self._login(vault) + + token_file = isolated_home / ".litellm" / "token.json" + assert result.exit_code == 0 + assert "Credential stored in your OS keychain." in result.output + assert json.loads(vault.blob)["key"] == "sk-minted" + assert "sk-minted" not in token_file.read_text() + assert json.loads(token_file.read_text())["user_id"] == "test-user-123" + + def test_login_without_a_keychain_says_where_the_credential_went(self, isolated_home, secret_vault_factory): + result = self._login(secret_vault_factory(available=False)) + + token_file = isolated_home / ".litellm" / "token.json" + assert result.exit_code == 0 + assert "No OS keychain available" in result.output + assert str(token_file) in result.output + assert json.loads(token_file.read_text())["key"] == "sk-minted" + + def test_whoami_and_print_token_read_through_the_keychain(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + self._login(vault) + obj = {"base_url": "https://test.example.com", "secret_vault": vault} + + whoami_result = self.runner.invoke(whoami, obj=obj) + print_result = self.runner.invoke(print_token, obj=obj) + + assert "Authenticated" in whoami_result.output + assert "test-user-123" in whoami_result.output + assert print_result.exit_code == 0 + assert print_result.stdout.strip() == "sk-minted" + + def test_logout_clears_the_keychain_as_well_as_the_file(self, isolated_home, secret_vault_factory): + vault = secret_vault_factory() + self._login(vault) + + result = self.runner.invoke(logout, obj={"base_url": "https://test.example.com", "secret_vault": vault}) + + assert result.exit_code == 0 + assert "Logged out successfully" in result.output + assert vault.blob is None + assert not (isolated_home / ".litellm" / "token.json").exists() + + def test_logout_warns_when_the_keychain_will_not_release_the_secret(self, isolated_home, secret_vault_factory): + """Silently reporting success would leave a live credential in the keychain.""" + vault = secret_vault_factory(erasable=False) + self._login(vault) + + result = self.runner.invoke(logout, obj={"base_url": "https://test.example.com", "secret_vault": vault}) + + assert result.exit_code == 0 + assert "could not be removed" in result.output + assert not (isolated_home / ".litellm" / "token.json").exists() + + def test_print_token_explains_a_locked_keychain_instead_of_printing_nothing( + self, isolated_home, secret_vault_factory + ): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "https://test.example.com", "user_id": "u-1", "timestamp": time.time()}, + ) + obj = {"base_url": "https://test.example.com", "secret_vault": secret_vault_factory(available=False)} + + result = self.runner.invoke(print_token, obj=obj) + + assert result.exit_code == 1 + assert KEYCHAIN_UNREACHABLE_MESSAGE in result.output + + def test_whoami_flags_a_locked_keychain(self, isolated_home, secret_vault_factory): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "https://test.example.com", "user_id": "u-1", "timestamp": time.time()}, + ) + obj = {"base_url": "https://test.example.com", "secret_vault": secret_vault_factory(available=False)} + + result = self.runner.invoke(whoami, obj=obj) + + assert "Authenticated" in result.output + assert KEYCHAIN_UNREACHABLE_MESSAGE in result.output + + +class TestApiKeyPrecedence: + """`LITELLM_PROXY_API_KEY` and `--api-key` outrank the stored credential; moving the secret + into the keychain must not disturb that order.""" + + def _resolved_key(self, args, obj=None): + with patch("litellm.proxy.client.cli.main.print_version") as mock_print_version: + result = CliRunner().invoke(cli, [*args, "version"], obj=obj) + assert result.exit_code == 0, result.output + return mock_print_version.call_args[0][1] + + def test_the_stored_credential_is_the_fallback(self, isolated_home): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "http://localhost:4000", "key": "sk-stored", "timestamp": time.time()}, + ) + + assert self._resolved_key([]) == "sk-stored" + + def test_the_stored_credential_is_read_through_the_injected_keychain(self, isolated_home, secret_vault_factory): + """The vault handed to the CLI through ctx.obj must be the one the group callback reads, + so a keychain-held secret resolves without ever touching the host OS keychain.""" + _write_home_json(isolated_home, "token.json", {"base_url": "http://localhost:4000", "timestamp": time.time()}) + vault = secret_vault_factory(_secret_blob("http://localhost:4000", "sk-keychain")) + + assert self._resolved_key([], obj={"secret_vault": vault}) == "sk-keychain" + + def test_env_var_beats_the_stored_credential(self, isolated_home, monkeypatch): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "http://localhost:4000", "key": "sk-stored", "timestamp": time.time()}, + ) + monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-from-env") + + assert self._resolved_key([]) == "sk-from-env" + + def test_explicit_api_key_beats_both(self, isolated_home, monkeypatch): + _write_home_json( + isolated_home, + "token.json", + {"base_url": "http://localhost:4000", "key": "sk-stored", "timestamp": time.time()}, + ) + monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-from-env") + + assert self._resolved_key(["--api-key", "sk-explicit"]) == "sk-explicit" + + class TestLoginConfigClaude: """`lite login --config-claude` wiring into ~/.claude/settings.json""" @@ -1054,7 +1065,7 @@ class TestLoginConfigClaude: patch("webbrowser.open"), patch("requests.post", return_value=_mock_cli_sso_start_response()), patch("requests.get", return_value=poll_response), - patch("litellm.proxy.client.cli.commands.auth.save_token"), + patch("litellm.proxy.client.cli.commands.auth.save_cli_token"), patch("litellm.proxy.client.cli.interface.show_commands"), patch("litellm.proxy.client.cli.commands.auth.CLAUDE_SETTINGS_PATH", settings_path), patch( diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py index bc9744eb410..9010fb4c022 100644 --- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -7,6 +7,7 @@ from unittest.mock import patch import pytest from click.testing import CliRunner +from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.claude_settings import ( AUTOROUTE_BACKUP_PATH, @@ -181,18 +182,18 @@ class TestApiKeyHelperIsActuallyInvocable: assert result.exit_code != 2 def test_the_generated_command_reaches_print_token(self): - with patch(f"{AUTH_MODULE}.load_token", return_value=None): + with patch(f"{AUTH_MODULE}.load_cli_token", return_value=None): result = CliRunner().invoke(cli, self._helper_args("http://localhost:4000")) assert "Not authenticated" in result.output def test_the_generated_command_carries_the_base_url_through(self): - stale = { - "base_url": "http://other-proxy.example.com", - "key": "sk-stale", - "timestamp": time.time(), - } - with patch(f"{AUTH_MODULE}.load_token", return_value=stale): + stale = CliTokenRecord( + base_url="http://other-proxy.example.com", + key="sk-stale", + timestamp=time.time(), + ) + with patch(f"{AUTH_MODULE}.load_cli_token", return_value=stale): result = CliRunner().invoke(cli, self._helper_args("http://localhost:4000")) assert "Not authenticated for this server" in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_config_commands.py b/tests/test_litellm/proxy/client/cli/test_config_commands.py index d81ee6bd2b1..6f3f4e4b268 100644 --- a/tests/test_litellm/proxy/client/cli/test_config_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_config_commands.py @@ -18,7 +18,7 @@ from litellm.proxy.client.cli.commands.config import ( load_config, save_config, ) -from litellm.proxy.client.cli.commands.private_json import write_private_json +from litellm.litellm_core_utils.private_json import write_private_json from litellm.proxy.client.cli.interface import show_commands @@ -355,7 +355,7 @@ class TestWritePrivateJson: def _interrupt(*args: object, **kwargs: object) -> None: raise KeyboardInterrupt() - monkeypatch.setattr("litellm.proxy.client.cli.commands.private_json.json.dump", _interrupt) + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _interrupt) target = tmp_path / "config.json" with pytest.raises(KeyboardInterrupt): diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index 51de0dcf11d..aebf441f777 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -8,6 +8,7 @@ import click import pytest from click.testing import CliRunner +from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord from litellm.proxy.client.cli.commands import up as up_module from litellm.proxy.client.cli.commands.agents import AgentRunError from litellm.proxy.client.cli.commands.claude_settings import ClaudeSettingsError @@ -220,13 +221,17 @@ def _make_ctx(base_url): return click.Context(click.Command("test"), obj={"base_url": base_url}) +def _token(key, base_url): + return CliTokenRecord(key=key, base_url=base_url) + + class TestEnsureFreshLogin: """A token that is fresh but was issued for a *different* proxy must not be trusted: without this check, a user logged into proxy A who runs `up --base-url proxy-b` would silently get an apiKeyHelper wired up around proxy A's real token, which print-token would then hand to proxy B.""" def test_reuses_a_fresh_token_issued_for_the_same_proxy(self, monkeypatch): - monkeypatch.setattr(up_module, "load_token", lambda: {"key": "sk-a", "base_url": "http://proxy-a:4000"}) + monkeypatch.setattr(up_module, "load_cli_token", lambda **_: _token("sk-a", "http://proxy-a:4000")) monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) login_calls = [] monkeypatch.setattr(up_module, "login", lambda ctx: login_calls.append(ctx)) @@ -239,11 +244,11 @@ class TestEnsureFreshLogin: monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: True) tokens = iter( [ - {"key": "sk-a", "base_url": "http://proxy-a:4000"}, - {"key": "sk-b", "base_url": "http://proxy-b:4000"}, + _token("sk-a", "http://proxy-a:4000"), + _token("sk-b", "http://proxy-b:4000"), ] ) - monkeypatch.setattr(up_module, "load_token", lambda: next(tokens)) + monkeypatch.setattr(up_module, "load_cli_token", lambda **_: next(tokens)) monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) login_calls = [] @@ -259,7 +264,7 @@ class TestEnsureFreshLogin: def test_fails_cleanly_non_interactively_when_only_a_different_proxys_token_is_cached(self, monkeypatch): monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: False) - monkeypatch.setattr(up_module, "load_token", lambda: {"key": "sk-a", "base_url": "http://proxy-a:4000"}) + monkeypatch.setattr(up_module, "load_cli_token", lambda **_: _token("sk-a", "http://proxy-a:4000")) monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) with pytest.raises(UpError, match="lite login"): @@ -276,7 +281,7 @@ class TestUpCommand: backup_path.write_text(json.dumps(existing_backup)) with ( - patch(f"{UP_MODULE}.load_token", return_value={"key": "sk-fresh", "base_url": "http://localhost:4000"}), + patch(f"{UP_MODULE}.load_cli_token", return_value=_token("sk-fresh", "http://localhost:4000")), patch(f"{UP_MODULE}.is_cli_token_fresh", return_value=True), patch(f"{UP_MODULE}.resolve_api_key", return_value="sk-fresh"), patch(f"{UP_MODULE}.verify_proxy_key"), @@ -293,7 +298,7 @@ class TestUpCommand: _patch_paths(monkeypatch, tmp_path) monkeypatch.setattr(sys.stdin, "isatty", lambda: False) - with patch(f"{UP_MODULE}.load_token", return_value=None): + with patch(f"{UP_MODULE}.load_cli_token", return_value=None): result = self.runner.invoke(up, obj={"base_url": "http://localhost:4000"}) assert result.exit_code != 0 @@ -303,7 +308,7 @@ class TestUpCommand: _patch_paths(monkeypatch, tmp_path) with ( - patch(f"{UP_MODULE}.load_token", return_value={"key": "sk-fresh", "base_url": "http://localhost:4000"}), + patch(f"{UP_MODULE}.load_cli_token", return_value=_token("sk-fresh", "http://localhost:4000")), patch(f"{UP_MODULE}.is_cli_token_fresh", return_value=True), patch(f"{UP_MODULE}.resolve_api_key", return_value="sk-fresh"), patch( @@ -329,7 +334,7 @@ class TestUpCommand: return True with ( - patch(f"{UP_MODULE}.load_token", return_value={"key": "sk-fresh", "base_url": "http://localhost:4000"}), + patch(f"{UP_MODULE}.load_cli_token", return_value=_token("sk-fresh", "http://localhost:4000")), patch(f"{UP_MODULE}.is_cli_token_fresh", return_value=True), patch(f"{UP_MODULE}.resolve_api_key", return_value="sk-fresh"), patch(f"{UP_MODULE}.verify_proxy_key"), diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 31726acfbaa..a5c5a9f135b 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22806 + "limit": 22805 }, "LIT002": { - "limit": 26878 + "limit": 26877 }, "LIT003": { "limit": 269 @@ -27,12 +27,12 @@ "limit": 0 }, "LIT010": { - "limit": 16695 + "limit": 16693 }, "LIT011": { "limit": 5588 }, "LIT012": { - "limit": 4519 + "limit": 4511 } } diff --git a/uv.lock b/uv.lock index d9e2fb94667..53bb0cb8f82 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-16T00:41:08.185444Z" +exclude-newer = "2026-08-17T01:06:38.502388Z" exclude-newer-span = "P3D" [manifest] @@ -710,6 +710,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, ] +[[package]] +name = "backports-tarfile" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", size = 86406, upload-time = "2024-05-28T17:01:54.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, +] + [[package]] name = "basedpyright" version = "1.39.7" @@ -3538,6 +3547,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, ] +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-tarfile", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, +] + +[[package]] +name = "jaraco-functools" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/1f/c23395957d41ccf27c4e535c3d334c4051e5395b3752057ba4cbaec35c56/jaraco_functools-4.6.0.tar.gz", hash = "sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280", size = 20837, upload-time = "2026-07-14T01:28:02.544Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/36/ecc85bc96c273dc8a11273ed4782272975e6338d4a3e9228621175edf0e3/jaraco_functools-4.6.0-py3-none-any.whl", hash = "sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30", size = 11677, upload-time = "2026-07-14T01:28:01.59Z" }, +] + +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -3764,6 +3818,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "keyring" +version = "25.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata", marker = "python_full_version < '3.12'" }, + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, +] + [[package]] name = "kiwisolver" version = "1.5.0" @@ -4222,6 +4294,7 @@ caching = [ ] cli = [ { name = "inquirerpy" }, + { name = "keyring" }, { name = "pyyaml" }, { name = "requests" }, { name = "rich" }, @@ -4352,6 +4425,7 @@ dev = [ { name = "diff-cover" }, { name = "fakeredis" }, { name = "fastapi-offline" }, + { name = "keyring" }, { name = "langfuse" }, { name = "openapi-core" }, { name = "opentelemetry-api" }, @@ -4446,6 +4520,7 @@ requires-dist = [ { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" }, { name = "jinja2", specifier = ">=3.1.6,<4.0" }, { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, + { name = "keyring", marker = "extra == 'cli'", specifier = ">=25.6.0,<26.0" }, { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = ">=2.59.7,<3.0" }, { name = "litellm-enterprise", marker = "extra == 'proxy'", editable = "enterprise" }, { name = "litellm-proxy-extras", marker = "extra == 'proxy'", editable = "litellm-proxy-extras" }, @@ -4532,6 +4607,7 @@ dev = [ { name = "diff-cover", specifier = "==9.7.2" }, { name = "fakeredis", specifier = "==2.34.1" }, { name = "fastapi-offline", specifier = "==1.7.6" }, + { name = "keyring", specifier = "==25.7.0" }, { name = "langfuse", specifier = "==2.59.7" }, { name = "openapi-core", specifier = "==0.22.0" }, { name = "opentelemetry-api", specifier = "==1.28.0" }, @@ -7790,6 +7866,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" }, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -8632,6 +8717,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, ] +[[package]] +name = "secretstorage" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jeepney" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, +] + [[package]] name = "semantic-router" version = "0.1.15" From bd322ed8a7eb6968b2af8bebce28eb9f19251fd3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:11:01 -0700 Subject: [PATCH 022/226] refactor(cli): state the credential-store precedence rules as contracts Drop the inline notes on keychain erasure and disk-vs-vault precedence in favour of docstrings on the two functions that own those rules, and remove a stale section header and a field note that the code already says plainly. --- litellm/litellm_core_utils/cli_keyring.py | 6 +++++- litellm/litellm_core_utils/cli_token_utils.py | 6 +++++- litellm/proxy/client/cli/commands/auth.py | 3 --- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index 873db64a728..fcbf5ada55a 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -98,10 +98,14 @@ class KeyringVault: return True def erase(self) -> bool: + """Whether the keychain is guaranteed to hold no credential afterwards. + + An uninstalled `keyring` package can never have stored one. A kill switch set after + a credential was stored leaves that entry out of reach, so erasure cannot be promised. + """ if _import_keyring() is None: return True if _keyring_disabled(): - # a credential stored before the kill switch was set may still be in the keychain return False match self.read(): case SecretUnavailable(): diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 9960192180c..dd263ccd412 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -168,8 +168,12 @@ def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecor def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) -> CliTokenRecord | None: + """Resolve the credential when both stores hold one. + + A secret still on disk is the fresher of the two, because it is only left there when the + keychain write that should have removed it failed, so it outranks the vault entry. + """ if record.key is not None: - # a secret still on disk means the last keychain write failed: the file outranks the vault return _migrate_file_secret(record, vault) try: secret: Final = CliTokenSecret.model_validate_json(blob) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 8b9ef5633da..c2b5b6a620f 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -76,7 +76,6 @@ KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( ) -# Token storage utilities def context_secret_vault(ctx: click.Context) -> SecretVault: """Where this invocation reads and writes secret material; injectable through ctx.obj for tests""" ctx_obj: Final[CliContextObj | None] = ctx.obj @@ -666,8 +665,6 @@ def login(ctx: click.Context, config_claude: bool): api_key: Final = auth_result["api_key"] user_id: Final = auth_result["user_id"] - # base_url is stored so we can verify origin before reusing the - # key on a subsequent CLI invocation. record: Final = CliTokenRecord( base_url=base_url.rstrip("/"), key=api_key, From 01add582982a253c2fff3466b608c1d0409ed1ae Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:45:38 -0700 Subject: [PATCH 023/226] fix(cli): name why a login fell back to the token file lite ships with every install of litellm, but the keyring package it needs for keychain storage only ships with the cli extra. Such a user on a Mac was told 'No OS keychain available' about a machine that plainly has one, with nothing pointing at the missing package. The vault now reports which of the three unusable states it is in, so login can point at the install, name the kill switch, or report a genuinely absent keychain. --- litellm/litellm_core_utils/cli_keyring.py | 56 +++++++++++++------ litellm/litellm_core_utils/cli_token_utils.py | 26 +++++---- litellm/proxy/client/README.md | 2 +- litellm/proxy/client/cli/commands/auth.py | 41 +++++++++++--- tests/test_litellm/conftest.py | 18 ++++-- .../test_cli_token_utils.py | 23 ++++---- .../proxy/client/cli/test_auth_commands.py | 28 ++++++++++ 7 files changed, 141 insertions(+), 53 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index fcbf5ada55a..b19b3d3bc83 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -5,9 +5,9 @@ SDK-level access to the OS keychain (macOS Keychain, Windows Credential Manager, Linux Secret Service) that holds the credential minted by `lite login`. The `keyring` package is optional and imported lazily, so importing this module -never pulls it in. Every failure is returned as a value: a machine with no -keychain, or one whose keychain is locked, must degrade to the token file rather -than break `lite` or the SDK. +never pulls it in. Every failure is returned as a value, naming which of the +three ways the keychain can be out of reach applies, so callers can degrade to +the token file and tell the user what to do about it. """ import os @@ -32,11 +32,28 @@ class SecretMissing: @dataclass(frozen=True, slots=True) -class SecretUnavailable: +class SecretStored: pass -SecretRead: TypeAlias = SecretFound | SecretMissing | SecretUnavailable +@dataclass(frozen=True, slots=True) +class KeyringNotInstalled: + pass + + +@dataclass(frozen=True, slots=True) +class KeyringDisabled: + pass + + +@dataclass(frozen=True, slots=True) +class KeyringUnreachable: + pass + + +KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable +SecretRead: TypeAlias = SecretFound | SecretMissing | KeyringUnusable +SecretWrite: TypeAlias = SecretStored | KeyringUnusable class SecretVault(Protocol): @@ -44,7 +61,7 @@ class SecretVault(Protocol): def read(self) -> SecretRead: ... - def write(self, blob: str) -> bool: ... + def write(self, blob: str) -> SecretWrite: ... def erase(self) -> bool: ... @@ -69,8 +86,11 @@ def _import_keyring() -> KeyringApi | None: return keyring -def _keyring_api() -> KeyringApi | None: - return None if _keyring_disabled() else _import_keyring() +def _keyring_api() -> KeyringApi | KeyringNotInstalled | KeyringDisabled: + if _keyring_disabled(): + return KeyringDisabled() + api: Final = _import_keyring() + return KeyringNotInstalled() if api is None else api @dataclass(frozen=True, slots=True) @@ -79,23 +99,23 @@ class KeyringVault: def read(self) -> SecretRead: api: Final = _keyring_api() - if api is None: - return SecretUnavailable() + if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): + return api try: blob: Final = api.get_password(KEYRING_SERVICE, KEYRING_ACCOUNT) except Exception: # noqa: BLE001 # backends raise outside keyring.errors; never break the SDK - return SecretUnavailable() + return KeyringUnreachable() return SecretMissing() if blob is None else SecretFound(blob) - def write(self, blob: str) -> bool: + def write(self, blob: str) -> SecretWrite: api: Final = _keyring_api() - if api is None: - return False + if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): + return api try: api.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, blob) except Exception: # noqa: BLE001 # a keychain that refuses the write falls back to the token file - return False - return True + return KeyringUnreachable() + return SecretStored() def erase(self) -> bool: """Whether the keychain is guaranteed to hold no credential afterwards. @@ -108,7 +128,7 @@ class KeyringVault: if _keyring_disabled(): return False match self.read(): - case SecretUnavailable(): + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): return False case SecretMissing(): return True @@ -117,7 +137,7 @@ class KeyringVault: def _delete(self) -> bool: api: Final = _keyring_api() - if api is None: + if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): return False try: api.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index dd263ccd412..40822e5b335 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -22,10 +22,14 @@ from pydantic import BaseModel, ConfigDict, ValidationError from litellm.litellm_core_utils.cli_keyring import ( SYSTEM_KEYRING, + KeyringDisabled, + KeyringNotInstalled, + KeyringUnreachable, SecretFound, SecretMissing, - SecretUnavailable, + SecretStored, SecretVault, + SecretWrite, ) from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json @@ -79,13 +83,15 @@ def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | N return _resolve_secret(record, vault) -def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> bool: - """Store a freshly minted credential. Returns whether the keychain took the secret""" - if record.key is None or not vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)): - _write_token_file(record) - return False - _write_token_file(_without_secret(record)) - return True +def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretWrite: + """Store a freshly minted credential. Reports whether the keychain took the secret, and why not""" + outcome: Final = ( + SecretStored() + if record.key is None + else vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)) + ) + _write_token_file(_without_secret(record) if isinstance(outcome, SecretStored) else record) + return outcome def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> bool: @@ -163,7 +169,7 @@ def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecor return _apply_vault_secret(record, blob, vault) case SecretMissing(): return _migrate_file_secret(record, vault) - case SecretUnavailable(): + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): return record @@ -188,7 +194,7 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) - def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: if record.key is None: return None - if vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)): + if isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): _scrub_file_secret(record) return record diff --git a/litellm/proxy/client/README.md b/litellm/proxy/client/README.md index 9ece4c2be3d..d46c7174b4c 100644 --- a/litellm/proxy/client/README.md +++ b/litellm/proxy/client/README.md @@ -377,7 +377,7 @@ The key itself goes into the OS keychain (macOS Keychain, Windows Credential Man } ``` -Headless boxes and CI runners usually have no keychain. There the key stays in the same `0600` file alongside the metadata, exactly as it did before, and `lite login` tells you which of the two happened. Set `LITELLM_CLI_DISABLE_KEYRING=1` to force the file even where a keychain exists. A `token.json` written by an older `lite` keeps working and is moved into the keychain, and scrubbed from the file, the first time a keychain-capable `lite` reads it. +Keychain storage needs the `keyring` package, which ships with `pip install 'litellm[cli]'`. Headless boxes and CI runners usually have no keychain either. In all of those cases the key stays in the same `0600` file alongside the metadata, exactly as it did before, and `lite login` names which one applies: the package is missing, the machine has no keychain, or you set `LITELLM_CLI_DISABLE_KEYRING=1` to force the file even where a keychain exists. A `token.json` written by an older `lite` keeps working and is moved into the keychain, and scrubbed from the file, the first time a keychain-capable `lite` reads it. `lite logout` clears both stores. If the keychain is locked at that moment it says so, and re-running it once the keychain is unlocked finishes the job. diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index c2b5b6a620f..03906f9b6df 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -11,7 +11,16 @@ from rich.table import Table from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import CLI_JWT_EXPIRATION_HOURS -from litellm.litellm_core_utils.cli_keyring import SYSTEM_KEYRING, SecretVault +from litellm.litellm_core_utils.cli_keyring import ( + DISABLE_KEYRING_ENV_VAR, + SYSTEM_KEYRING, + KeyringDisabled, + KeyringNotInstalled, + KeyringUnreachable, + SecretStored, + SecretVault, + SecretWrite, +) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, clear_cli_token, @@ -72,9 +81,29 @@ class CliAuthResult(TypedDict): KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( - "Your credential is stored in your OS keychain, which could not be read. Unlock it and retry, or run 'lite login'." + "Your credential is stored in your OS keychain, which could not be read. Unlock it and retry, " + "install the keyring package with: pip install 'litellm[cli]', or run 'lite login'." ) +KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'" + + +def storage_notice(outcome: SecretWrite) -> str: + """Tell the user where the credential ended up, and how to get keychain storage if it did not.""" + path: Final = get_cli_token_file_path() + match outcome: + case SecretStored(): + return "Credential stored in your OS keychain." + case KeyringNotInstalled(): + return ( + f"Credential stored in {path} (owner-only). " + f"For OS keychain storage, install the keyring package with: {KEYRING_INSTALL_HINT}" + ) + case KeyringDisabled(): + return f"Keychain storage is off ({DISABLE_KEYRING_ENV_VAR}). Credential stored in {path} (owner-only)." + case KeyringUnreachable(): + return f"No OS keychain available. Credential stored in {path} (owner-only)." + def context_secret_vault(ctx: click.Context) -> SecretVault: """Where this invocation reads and writes secret material; injectable through ctx.obj for tests""" @@ -675,15 +704,11 @@ def login(ctx: click.Context, config_claude: bool): jwt_token="", timestamp=time.time(), ) - in_keychain: Final = save_cli_token(record, vault=context_secret_vault(ctx)) + stored: Final = save_cli_token(record, vault=context_secret_vault(ctx)) click.echo("\nLogin successful!") click.echo(f"JWT Token: {api_key[:20]}...") - click.echo( - "Credential stored in your OS keychain." - if in_keychain - else f"No OS keychain available; credential stored in {get_cli_token_file_path()} (owner-only)." - ) + click.echo(storage_notice(stored)) click.echo("You can now use the CLI without specifying --api-key") if config_claude: diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index ce0fd197538..a716ec0e0aa 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -23,10 +23,13 @@ from litellm import router as litellm_router_module from litellm import utils as litellm_utils_module from litellm._logging import ALL_LOGGERS from litellm.litellm_core_utils.cli_keyring import ( + KeyringUnreachable, + KeyringUnusable, SecretFound, SecretMissing, SecretRead, - SecretUnavailable, + SecretStored, + SecretWrite, ) from litellm.litellm_core_utils.prompt_templates import ( image_handling as image_handling_module, @@ -125,7 +128,8 @@ class FakeSecretVault: """In-memory stand-in for the OS keychain, injected wherever CLI credential storage is exercised. `available=False` models a keychain that is locked or has no backend, `writable=False` one that - refuses to store, and `erasable=False` one that will not release what it already holds. + refuses to store, `erasable=False` one that will not release what it already holds, and `failure` + picks which unusable state those report. """ def __init__( @@ -135,11 +139,13 @@ class FakeSecretVault: available: bool = True, writable: bool = True, erasable: bool = True, + failure: KeyringUnusable = KeyringUnreachable(), ) -> None: self.blob: str | None = blob self.available: bool = available self.writable: bool = writable self.erasable: bool = erasable + self.failure: KeyringUnusable = failure self.reads: int = 0 self.writes: list[str] = [] self.erases: int = 0 @@ -147,15 +153,15 @@ class FakeSecretVault: def read(self) -> SecretRead: self.reads += 1 if not self.available: - return SecretUnavailable() + return self.failure return SecretMissing() if self.blob is None else SecretFound(self.blob) - def write(self, blob: str) -> bool: + def write(self, blob: str) -> SecretWrite: self.writes.append(blob) if not (self.available and self.writable): - return False + return self.failure self.blob = blob - return True + return SecretStored() def erase(self) -> bool: self.erases += 1 diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 56ab6bcbfe0..961d986e8f5 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -13,7 +13,10 @@ from litellm.litellm_core_utils.cli_keyring import ( KeyringVault, SecretFound, SecretMissing, - SecretUnavailable, + KeyringDisabled, + KeyringNotInstalled, + KeyringUnreachable, + SecretStored, ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, @@ -253,7 +256,7 @@ class TestSaveCliToken: vault=vault, ) - assert stored is True + assert stored == SecretStored() assert "sk-new" not in _token_file(isolated_home).read_text() assert json.loads(vault.blob)["key"] == "sk-new" assert load_cli_token(vault=vault).key == "sk-new" @@ -265,7 +268,7 @@ class TestSaveCliToken: ) path = _token_file(isolated_home) - assert stored is False + assert stored == KeyringUnreachable() assert json.loads(path.read_text())["key"] == "sk-new" assert stat.S_IMODE(path.stat().st_mode) == 0o600 assert list(path.parent.glob(".tmp-*")) == [] @@ -379,7 +382,7 @@ class TestKeyringVault: fake = install_fake_keyring(_FakeKeyringModule()) vault = KeyringVault() - assert vault.write("blob-1") is True + assert vault.write("blob-1") == SecretStored() assert vault.read() == SecretFound("blob-1") assert vault.erase() is True assert vault.read() == SecretMissing() @@ -393,8 +396,8 @@ class TestKeyringVault: monkeypatch.setenv(DISABLE_KEYRING_ENV_VAR, "1") vault = KeyringVault() - assert vault.read() == SecretUnavailable() - assert vault.write("blob-1") is False + assert vault.read() == KeyringDisabled() + assert vault.write("blob-1") == KeyringDisabled() assert vault.erase() is False def test_an_uninstalled_keyring_library_degrades_to_the_file(self, monkeypatch): @@ -404,19 +407,19 @@ class TestKeyringVault: monkeypatch.setitem(sys.modules, "keyring", None) vault = KeyringVault() - assert vault.read() == SecretUnavailable() - assert vault.write("blob-1") is False + assert vault.read() == KeyringNotInstalled() + assert vault.write("blob-1") == KeyringNotInstalled() assert vault.erase() is True def test_a_locked_keychain_is_reported_not_raised(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("keyring is locked"))) - assert KeyringVault().read() == SecretUnavailable() + assert KeyringVault().read() == KeyringUnreachable() def test_a_refused_write_is_reported_not_raised(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(set_error=RuntimeError("no backend"))) - assert KeyringVault().write("blob-1") is False + assert KeyringVault().write("blob-1") == KeyringUnreachable() def test_a_refused_delete_is_reported_so_logout_can_warn(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(stored="blob-1", delete_error=RuntimeError("locked"))) diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index e93f05cb4aa..1f2d48f6547 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -13,6 +13,11 @@ import pytest from click.testing import CliRunner from litellm.constants import CLI_JWT_EXPIRATION_HOURS +from litellm.litellm_core_utils.cli_keyring import ( + DISABLE_KEYRING_ENV_VAR, + KeyringDisabled, + KeyringNotInstalled, +) from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord, save_cli_token from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.auth import ( @@ -931,6 +936,29 @@ class TestKeychainBackedCommands: assert str(token_file) in result.output assert json.loads(token_file.read_text())["key"] == "sk-minted" + def test_login_points_a_user_missing_the_keyring_package_at_the_install( + self, isolated_home, secret_vault_factory + ): + """`lite` ships with every install, the keyring package only with the cli extra. Telling + that user their machine has no keychain sends them looking for a problem they do not have.""" + result = self._login(secret_vault_factory(available=False, failure=KeyringNotInstalled())) + + token_file = isolated_home / ".litellm" / "token.json" + assert result.exit_code == 0 + assert "pip install 'litellm[cli]'" in result.output + assert "No OS keychain available" not in result.output + assert json.loads(token_file.read_text())["key"] == "sk-minted" + + def test_login_names_the_kill_switch_instead_of_blaming_the_machine( + self, isolated_home, secret_vault_factory + ): + result = self._login(secret_vault_factory(available=False, failure=KeyringDisabled())) + + assert result.exit_code == 0 + assert DISABLE_KEYRING_ENV_VAR in result.output + assert "No OS keychain available" not in result.output + assert json.loads((isolated_home / ".litellm" / "token.json").read_text())["key"] == "sk-minted" + def test_whoami_and_print_token_read_through_the_keychain(self, isolated_home, secret_vault_factory): vault = secret_vault_factory() self._login(vault) From 1750893a6905f45d7fa9cc65167856ae6c182208 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:26:41 -0700 Subject: [PATCH 024/226] fix(cli): never report success while a credential is still readable Migration moved the secret into the keychain and then suppressed any OSError from rewriting token.json, so a file that could not be rewritten kept the credential in cleartext while every command reported success. That file is now removed instead: signing in again costs one command, a stranded live credential costs the credential `lite logout` also reported a clean logout whenever the keyring package was missing, on the reasoning that an install without it could never have stored anything. The entry belongs to the OS, so a keychain-backed login survives a logout run from a venv without the cli extra. erase() now reports which keychain state applies, and logout warns with the advice that fixes each one, staying quiet for file-backed logins whose token file still carries its own secret Also pins the migration path's tightening of a world-readable legacy token.json, and moves the logout tests off patch() onto the injected vault --- litellm/litellm_core_utils/cli_keyring.py | 41 ++++++---- litellm/litellm_core_utils/cli_token_utils.py | 49 ++++++++++-- litellm/proxy/client/cli/commands/auth.py | 31 +++++--- tests/test_litellm/conftest.py | 13 +++- .../test_cli_token_utils.py | 77 ++++++++++++++++--- .../proxy/client/cli/test_auth_commands.py | 66 ++++++++++++++-- 6 files changed, 226 insertions(+), 51 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index b19b3d3bc83..0497991c2a0 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -36,6 +36,16 @@ class SecretStored: pass +@dataclass(frozen=True, slots=True) +class SecretErased: + pass + + +@dataclass(frozen=True, slots=True) +class SecretStranded: + pass + + @dataclass(frozen=True, slots=True) class KeyringNotInstalled: pass @@ -54,6 +64,7 @@ class KeyringUnreachable: KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable SecretRead: TypeAlias = SecretFound | SecretMissing | KeyringUnusable SecretWrite: TypeAlias = SecretStored | KeyringUnusable +SecretErase: TypeAlias = SecretErased | SecretStranded | KeyringUnusable class SecretVault(Protocol): @@ -63,7 +74,7 @@ class SecretVault(Protocol): def write(self, blob: str) -> SecretWrite: ... - def erase(self) -> bool: ... + def erase(self) -> SecretErase: ... class KeyringApi(Protocol): @@ -117,33 +128,31 @@ class KeyringVault: return KeyringUnreachable() return SecretStored() - def erase(self) -> bool: - """Whether the keychain is guaranteed to hold no credential afterwards. + def erase(self) -> SecretErase: + """Remove our entry, reporting whether the keychain is guaranteed to be free of it. - An uninstalled `keyring` package can never have stored one. A kill switch set after - a credential was stored leaves that entry out of reach, so erasure cannot be promised. + A keychain out of reach is never an erasure: the entry belongs to the OS, not to this + install, so it outlives an uninstalled `keyring` package and a kill switch set after login. + Those cases are reported apart from a confirmed entry that would not delete, because only + the caller knows whether this machine ever put a secret in a keychain. """ - if _import_keyring() is None: - return True - if _keyring_disabled(): - return False match self.read(): - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): - return False + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() as unusable: + return unusable case SecretMissing(): - return True + return SecretErased() case SecretFound(): return self._delete() - def _delete(self) -> bool: + def _delete(self) -> SecretErase: api: Final = _keyring_api() if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): - return False + return api try: api.delete_password(KEYRING_SERVICE, KEYRING_ACCOUNT) except Exception: # noqa: BLE001 # report the failure as a value so `lite logout` can warn - return False - return True + return SecretStranded() + return SecretErased() SYSTEM_KEYRING: Final[SecretVault] = KeyringVault() diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 40822e5b335..cd69f9470a3 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -25,9 +25,12 @@ from litellm.litellm_core_utils.cli_keyring import ( KeyringDisabled, KeyringNotInstalled, KeyringUnreachable, + SecretErase, + SecretErased, SecretFound, SecretMissing, SecretStored, + SecretStranded, SecretVault, SecretWrite, ) @@ -84,7 +87,7 @@ def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | N def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretWrite: - """Store a freshly minted credential. Reports whether the keychain took the secret, and why not""" + """Store a freshly minted credential. Reports where its secret material ended up, and why""" outcome: Final = ( SecretStored() if record.key is None @@ -94,11 +97,33 @@ def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRIN return outcome -def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> bool: - """Remove the credential from both stores. Returns whether the keychain is now free of it""" - erased: Final = vault.erase() +def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretErase: + """Remove the credential from both stores. Reports whether the keychain is now free of it""" + outcome: Final = vault.erase() + settled: Final = _nothing_left_behind(outcome) Path(get_cli_token_file_path()).unlink(missing_ok=True) - return erased + return SecretErased() if settled else outcome + + +def _nothing_left_behind(outcome: SecretErase) -> bool: + """Whether the keychain can be trusted to hold no credential of ours once the file is gone""" + match outcome: + case SecretErased(): + return True + case SecretStranded(): + return False + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): + return not _secret_lives_in_keychain() + + +def _secret_lives_in_keychain() -> bool: + """Whether the token file is the metadata half of a pair whose secret half went to a keychain. + + A file that still carries its own secret rules one out, which keeps `lite logout` quiet on the + machines that never had a keychain to begin with. + """ + record: Final = _read_token_file() + return record is not None and record.key is None and not record.jwt_token def get_litellm_gateway_api_key( @@ -200,10 +225,22 @@ def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliToken def _scrub_file_secret(record: CliTokenRecord) -> None: + """Leave no secret material in the token file once the vault holds it. + + A file that cannot be rewritten without the secret is removed instead. Signing in again costs + the user one command; a live credential left behind in cleartext costs them the credential. + """ if record.key is None and not record.jwt_token: return - with contextlib.suppress(OSError): + try: _write_token_file(_without_secret(record)) + except OSError: + _discard_token_file() + + +def _discard_token_file() -> None: + with contextlib.suppress(OSError): + Path(get_cli_token_file_path()).unlink(missing_ok=True) def _without_secret(record: CliTokenRecord) -> CliTokenRecord: diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 03906f9b6df..4cf18435473 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -17,7 +17,9 @@ from litellm.litellm_core_utils.cli_keyring import ( KeyringDisabled, KeyringNotInstalled, KeyringUnreachable, + SecretErased, SecretStored, + SecretStranded, SecretVault, SecretWrite, ) @@ -80,12 +82,16 @@ class CliAuthResult(TypedDict): team_id: str | None -KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( - "Your credential is stored in your OS keychain, which could not be read. Unlock it and retry, " - "install the keyring package with: pip install 'litellm[cli]', or run 'lite login'." +KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'" + +STRANDED_CREDENTIAL_MESSAGE: Final = ( + "Logged out locally, but your credential is still in the OS keychain and could not be removed." ) -KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'" +KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( + "Your credential is stored in your OS keychain, which could not be read. Unlock it, or install " + f"the keyring package with: {KEYRING_INSTALL_HINT}. Run 'lite login' to start over." +) def storage_notice(outcome: SecretWrite) -> str: @@ -742,11 +748,18 @@ def login(ctx: click.Context, config_claude: bool): @click.pass_context def logout(ctx: click.Context): """Logout and clear stored authentication""" - if clear_cli_token(vault=context_secret_vault(ctx)): - click.echo("Logged out successfully. Authentication token cleared.") - return - click.echo("Logged out. The local token file is gone, but the OS keychain entry could not be removed.") - click.echo("Unlock your keychain and run 'lite logout' again to clear it.") + match clear_cli_token(vault=context_secret_vault(ctx)): + case SecretErased(): + click.echo("Logged out successfully. Authentication token cleared.") + case KeyringNotInstalled(): + click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo(f"Install the keyring package with: {KEYRING_INSTALL_HINT}, then run 'lite logout' again.") + case KeyringDisabled(): + click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo(f"Unset {DISABLE_KEYRING_ENV_VAR} and run 'lite logout' again to clear it.") + case SecretStranded() | KeyringUnreachable(): + click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo("Unlock your keychain and run 'lite logout' again to clear it.") @click.command(name="print-token") diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index a716ec0e0aa..b42355fa045 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -25,10 +25,13 @@ from litellm._logging import ALL_LOGGERS from litellm.litellm_core_utils.cli_keyring import ( KeyringUnreachable, KeyringUnusable, + SecretErase, + SecretErased, SecretFound, SecretMissing, SecretRead, SecretStored, + SecretStranded, SecretWrite, ) from litellm.litellm_core_utils.prompt_templates import ( @@ -163,12 +166,14 @@ class FakeSecretVault: self.blob = blob return SecretStored() - def erase(self) -> bool: + def erase(self) -> SecretErase: self.erases += 1 - if not (self.available and self.erasable): - return False + if not self.available: + return self.failure + if not self.erasable: + return SecretStranded() if self.blob is not None else SecretErased() self.blob = None - return True + return SecretErased() @pytest.fixture diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 961d986e8f5..925b440dfb7 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -16,7 +16,9 @@ from litellm.litellm_core_utils.cli_keyring import ( KeyringDisabled, KeyringNotInstalled, KeyringUnreachable, + SecretErased, SecretStored, + SecretStranded, ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, @@ -126,6 +128,16 @@ class TestLoadCliToken: assert on_disk["user_email"] == "user@example.com" assert stat.S_IMODE(path.stat().st_mode) == 0o600 + def test_migration_tightens_a_world_readable_legacy_file(self, isolated_home, secret_vault_factory): + """An older `lite`, a loose umask, or a restored backup can leave token.json readable by + every account on the box. Migrating it must not preserve those permissions.""" + path = _write_legacy_file(isolated_home) + path.chmod(0o644) + + load_cli_token(vault=secret_vault_factory()) + + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + def test_legacy_file_survives_a_vault_that_refuses_to_store(self, isolated_home, secret_vault_factory): """Scrubbing the only copy of the secret after a failed keychain write would log the user out for good.""" @@ -304,12 +316,35 @@ class TestSaveCliToken: assert list(path.parent.glob(".tmp-*")) == [] +class TestScrubFailure: + """A keychain that took the secret while the file kept it is the worst of both stores: the + credential is live, it is in cleartext on disk, and every command reports success.""" + + def test_a_file_that_cannot_be_rewritten_is_removed_instead( + self, isolated_home, secret_vault_factory, monkeypatch + ): + path = _write_legacy_file(isolated_home) + vault = secret_vault_factory() + + def _explode(*args, **kwargs): + raise OSError("no space left on device") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + + record = load_cli_token(vault=vault) + + assert record.key == "sk-legacy" + assert json.loads(vault.blob)["key"] == "sk-legacy" + assert not path.exists() + assert list(path.parent.glob(".tmp-*")) == [] + + class TestClearCliToken: def test_removes_the_credential_from_both_stores(self, isolated_home, secret_vault_factory): vault = secret_vault_factory() save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) - assert clear_cli_token(vault=vault) is True + assert clear_cli_token(vault=vault) == SecretErased() assert vault.blob is None assert not _token_file(isolated_home).exists() assert load_cli_token(vault=vault) is None @@ -318,11 +353,32 @@ class TestClearCliToken: _write_legacy_file(isolated_home) vault = secret_vault_factory(blob=_blob(), erasable=False) - assert clear_cli_token(vault=vault) is False + assert clear_cli_token(vault=vault) == SecretStranded() + assert not _token_file(isolated_home).exists() + + def test_logout_from_an_install_without_keyring_does_not_claim_the_keychain_is_clear( + self, isolated_home, secret_vault_factory + ): + """Log in where `litellm[cli]` is installed and the secret goes to the OS keychain; log out + from a venv without it and the entry survives, because it belongs to the OS rather than to + the package. Reporting a clean logout there leaves a live credential the user thinks is gone.""" + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) + + assert clear_cli_token(vault=vault) == KeyringNotInstalled() + assert not _token_file(isolated_home).exists() + + def test_a_file_backed_login_logs_out_quietly_without_keyring(self, isolated_home, secret_vault_factory): + """The complement: a user who never had a keychain keeps their whole credential in the file, + so removing it is a complete logout and must not warn about an entry that cannot exist.""" + _write_legacy_file(isolated_home) + vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) + + assert clear_cli_token(vault=vault) == SecretErased() assert not _token_file(isolated_home).exists() def test_is_safe_when_nothing_was_ever_stored(self, isolated_home, secret_vault_factory): - assert clear_cli_token(vault=secret_vault_factory()) is True + assert clear_cli_token(vault=secret_vault_factory()) == SecretErased() class TestIsCliTokenFresh: @@ -384,7 +440,7 @@ class TestKeyringVault: assert vault.write("blob-1") == SecretStored() assert vault.read() == SecretFound("blob-1") - assert vault.erase() is True + assert vault.erase() == SecretErased() assert vault.read() == SecretMissing() assert {call[1:] for call in fake.calls} == {(KEYRING_SERVICE, KEYRING_ACCOUNT)} @@ -392,24 +448,25 @@ class TestKeyringVault: """`LITELLM_CLI_DISABLE_KEYRING` has to work without importing keyring, because keyring caches its backend on first use and cannot be reconfigured later. Erase still fails: a credential stored before the switch was set may be in the keychain, and with reads - disabled `lite logout` cannot verify it is gone, so it must warn instead.""" + disabled `lite logout` cannot verify it is gone, so it must say so instead.""" monkeypatch.setenv(DISABLE_KEYRING_ENV_VAR, "1") vault = KeyringVault() assert vault.read() == KeyringDisabled() assert vault.write("blob-1") == KeyringDisabled() - assert vault.erase() is False + assert vault.erase() == KeyringDisabled() def test_an_uninstalled_keyring_library_degrades_to_the_file(self, monkeypatch): """keyring is an optional extra, so the SDK must survive its absence rather than raise on - the hot path.""" + the hot path. Erase cannot succeed: the entry belongs to the OS and outlives the package, + so an install without it is not evidence that the keychain is empty.""" monkeypatch.delenv(DISABLE_KEYRING_ENV_VAR, raising=False) monkeypatch.setitem(sys.modules, "keyring", None) vault = KeyringVault() assert vault.read() == KeyringNotInstalled() assert vault.write("blob-1") == KeyringNotInstalled() - assert vault.erase() is True + assert vault.erase() == KeyringNotInstalled() def test_a_locked_keychain_is_reported_not_raised(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("keyring is locked"))) @@ -424,9 +481,9 @@ class TestKeyringVault: def test_a_refused_delete_is_reported_so_logout_can_warn(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(stored="blob-1", delete_error=RuntimeError("locked"))) - assert KeyringVault().erase() is False + assert KeyringVault().erase() == SecretStranded() def test_erasing_a_locked_keychain_is_a_failure(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("locked"))) - assert KeyringVault().erase() is False + assert KeyringVault().erase() == KeyringUnreachable() diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 1f2d48f6547..33bd8307c21 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -46,6 +46,12 @@ def _write_home_json(home: Path, filename: str, payload: dict[str, object]) -> N (litellm_dir / filename).write_text(json.dumps(payload)) +def _write_token_file(home: Path, *, key: str | None) -> None: + """A stored login: `key=None` is the metadata half of a keychain-backed pair, a key is a file-backed one.""" + payload: dict[str, object] = {"base_url": "https://test.example.com", "user_id": "u-1", "timestamp": time.time()} + _write_home_json(home, "token.json", payload if key is None else {**payload, "key": key}) + + def _secret_blob(base_url: str, key: str) -> str: return json.dumps({"base_url": base_url, "key": key, "jwt_token": ""}) @@ -427,14 +433,62 @@ class TestLogoutCommand: """Setup for each test""" self.runner = CliRunner() - def test_logout_success(self): + def test_logout_success(self, isolated_home, secret_vault_factory): """Test successful logout""" - with patch("litellm.proxy.client.cli.commands.auth.clear_cli_token") as mock_clear: - result = self.runner.invoke(logout) + vault = secret_vault_factory(blob=_secret_blob("https://test.example.com", "sk-stored")) + _write_token_file(isolated_home, key=None) - assert result.exit_code == 0 - assert "Logged out successfully" in result.output - mock_clear.assert_called_once() + result = self.runner.invoke(logout, obj={"secret_vault": vault}) + + assert result.exit_code == 0 + assert "Logged out successfully" in result.output + assert vault.blob is None + assert not (isolated_home / ".litellm" / "token.json").exists() + + def test_logout_without_the_keyring_package_does_not_claim_the_keychain_is_clear( + self, isolated_home, secret_vault_factory + ): + """Logging out from an install without the cli extra cannot touch an entry a keychain-backed + login left behind, so it must point at the package rather than report a clean logout.""" + _write_token_file(isolated_home, key=None) + + result = self.runner.invoke( + logout, obj={"secret_vault": secret_vault_factory(available=False, failure=KeyringNotInstalled())} + ) + + assert result.exit_code == 0 + assert "Logged out successfully" not in result.output + assert "still in the OS keychain" in result.output + assert "pip install 'litellm[cli]'" in result.output + + def test_logout_warns_when_the_keychain_refuses_to_release_the_entry( + self, isolated_home, secret_vault_factory + ): + """A locked keychain leaves a live credential behind that the user believes is gone.""" + vault = secret_vault_factory( + blob=_secret_blob("https://test.example.com", "sk-stored"), erasable=False + ) + _write_token_file(isolated_home, key=None) + + result = self.runner.invoke(logout, obj={"secret_vault": vault}) + + assert result.exit_code == 0 + assert "Logged out successfully" not in result.output + assert "still in the OS keychain" in result.output + assert "Unlock your keychain" in result.output + + def test_logout_from_a_file_only_login_stays_quiet(self, isolated_home, secret_vault_factory): + """The credential never went to a keychain, so removing the file is the whole logout and + warning about a keychain entry would send the user chasing one that cannot exist.""" + _write_token_file(isolated_home, key="sk-in-file") + + result = self.runner.invoke( + logout, obj={"secret_vault": secret_vault_factory(available=False, failure=KeyringNotInstalled())} + ) + + assert result.exit_code == 0 + assert "Logged out successfully" in result.output + assert "still in the OS keychain" not in result.output class TestWhoamiCommand: From 424e74ba9ffc9f33757d3c68f90d0fef2cda4079 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:31:43 -0700 Subject: [PATCH 025/226] fix(cli): roll the keychain write back when the plaintext copy cannot be removed Removing the file when it could not be rewritten covered a full disk, but not a ~/.litellm that permits neither the rewrite nor the delete, which is what a `sudo lite login` leaves behind. There the secret was copied into the keychain and kept in cleartext on disk, so migration widened exposure instead of narrowing it Migration now only keeps the vault copy if the file's copy is gone. When it is not, the write is rolled back and the user is left exactly as they were, logged in with one copy of the credential --- litellm/litellm_core_utils/cli_token_utils.py | 26 +++++++++++++------ .../test_cli_token_utils.py | 20 ++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index cd69f9470a3..53289770c1a 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -12,7 +12,6 @@ first time it reads one. This module has no dependencies on proxy code and can be safely imported at the SDK level. """ -import contextlib import time from pathlib import Path from types import MappingProxyType @@ -217,30 +216,41 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) - def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: + """Move a file-held secret into the vault, but only if the file's copy can be taken away. + + Migrating without scrubbing would leave the credential live in two stores instead of one, so a + file that will not give its copy up rolls the vault write back rather than widening exposure. + """ if record.key is None: return None - if isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): - _scrub_file_secret(record) + if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): + return record + if not _scrub_file_secret(record): + vault.erase() return record -def _scrub_file_secret(record: CliTokenRecord) -> None: +def _scrub_file_secret(record: CliTokenRecord) -> bool: """Leave no secret material in the token file once the vault holds it. A file that cannot be rewritten without the secret is removed instead. Signing in again costs the user one command; a live credential left behind in cleartext costs them the credential. """ if record.key is None and not record.jwt_token: - return + return True try: _write_token_file(_without_secret(record)) except OSError: - _discard_token_file() + return _discard_token_file() + return True -def _discard_token_file() -> None: - with contextlib.suppress(OSError): +def _discard_token_file() -> bool: + try: Path(get_cli_token_file_path()).unlink(missing_ok=True) + except OSError: + return False + return True def _without_secret(record: CliTokenRecord) -> CliTokenRecord: diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index 925b440dfb7..cbc93bbde71 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -1,4 +1,5 @@ import json +import os import stat import sys import time @@ -320,6 +321,25 @@ class TestScrubFailure: """A keychain that took the secret while the file kept it is the worst of both stores: the credential is live, it is in cleartext on disk, and every command reports success.""" + @pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") + def test_a_file_that_will_not_give_its_copy_up_rolls_the_vault_write_back( + self, isolated_home, secret_vault_factory + ): + """Handing the keychain a copy without taking the file's away leaves the credential live in + two stores instead of one. A directory that permits neither the rewrite nor the delete, a + root-owned ~/.litellm left behind by a `sudo lite login`, must widen nothing.""" + path = _write_legacy_file(isolated_home) + vault = secret_vault_factory() + path.parent.chmod(0o500) + try: + record = load_cli_token(vault=vault) + finally: + path.parent.chmod(0o700) + + assert record.key == "sk-legacy" + assert json.loads(path.read_text())["key"] == "sk-legacy" + assert vault.blob is None + def test_a_file_that_cannot_be_rewritten_is_removed_instead( self, isolated_home, secret_vault_factory, monkeypatch ): From 3c73a39877fa97db3dff4e22dc685bcb77fa4041 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:10:59 -0700 Subject: [PATCH 026/226] fix(cli): verify every credential store transition before reporting it done A keyring backend can accept a write and keep nothing. That is exactly what `keyring --disable` and PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring select, and it raises nothing to distinguish itself, so `lite login` was handing the credential to a black hole, scrubbing its own copy from token.json, and printing a success message over a login that no longer worked. Reading the value back is the only way to tell that backend apart from a keychain that really stored the secret. The same rule closes the rest of the gaps. A credential the token file will not record is taken back out of the keychain instead of being left live on a machine with no record of it, and is reported rather than raised. The migration stages its scrubbed file before the keychain is handed anything, so a directory that will not accept the rewrite stops the move rather than leaving the secret in two places. Logout no longer reads a key in the file as proof that the keychain is clear, which was never sound across two separate runs, and only draws that conclusion when the `keyring` package is missing outright, where nothing could have reached a keychain at all. --- litellm/litellm_core_utils/cli_keyring.py | 26 +++- litellm/litellm_core_utils/cli_token_utils.py | 104 +++++++++----- litellm/litellm_core_utils/private_json.py | 32 ++++- litellm/proxy/client/cli/commands/auth.py | 63 +++++++-- .../test_cli_token_utils.py | 131 ++++++++++++++++-- .../proxy/client/cli/test_auth_commands.py | 62 ++++++++- 6 files changed, 356 insertions(+), 62 deletions(-) diff --git a/litellm/litellm_core_utils/cli_keyring.py b/litellm/litellm_core_utils/cli_keyring.py index 0497991c2a0..15282fc522c 100644 --- a/litellm/litellm_core_utils/cli_keyring.py +++ b/litellm/litellm_core_utils/cli_keyring.py @@ -6,8 +6,12 @@ Linux Secret Service) that holds the credential minted by `lite login`. The `keyring` package is optional and imported lazily, so importing this module never pulls it in. Every failure is returned as a value, naming which of the -three ways the keychain can be out of reach applies, so callers can degrade to -the token file and tell the user what to do about it. +ways the keychain can be out of reach applies, so callers can degrade to the +token file and tell the user what to do about it. + +A write is only reported as stored once it has been read back, because keyring's +null backend, which `keyring --disable` and headless CI images both select, +accepts every write and keeps nothing. """ import os @@ -61,7 +65,12 @@ class KeyringUnreachable: pass -KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable +@dataclass(frozen=True, slots=True) +class KeyringDiscardsWrites: + pass + + +KeyringUnusable: TypeAlias = KeyringNotInstalled | KeyringDisabled | KeyringUnreachable | KeyringDiscardsWrites SecretRead: TypeAlias = SecretFound | SecretMissing | KeyringUnusable SecretWrite: TypeAlias = SecretStored | KeyringUnusable SecretErase: TypeAlias = SecretErased | SecretStranded | KeyringUnusable @@ -119,6 +128,13 @@ class KeyringVault: return SecretMissing() if blob is None else SecretFound(blob) def write(self, blob: str) -> SecretWrite: + """Store the secret, reporting stored only once the keychain hands the same bytes back. + + A backend that accepts writes and keeps nothing, which is exactly what `keyring --disable` + and `PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring` select, raises nothing to + distinguish itself. Reading the value back is the only way to tell it apart from a keychain + that really stored the credential, and the caller is about to drop its own copy on our word. + """ api: Final = _keyring_api() if isinstance(api, (KeyringNotInstalled, KeyringDisabled)): return api @@ -126,7 +142,7 @@ class KeyringVault: api.set_password(KEYRING_SERVICE, KEYRING_ACCOUNT, blob) except Exception: # noqa: BLE001 # a keychain that refuses the write falls back to the token file return KeyringUnreachable() - return SecretStored() + return SecretStored() if self.read() == SecretFound(blob) else KeyringDiscardsWrites() def erase(self) -> SecretErase: """Remove our entry, reporting whether the keychain is guaranteed to be free of it. @@ -137,7 +153,7 @@ class KeyringVault: the caller knows whether this machine ever put a secret in a keychain. """ match self.read(): - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() as unusable: + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites() as unusable: return unusable case SecretMissing(): return SecretErased() diff --git a/litellm/litellm_core_utils/cli_token_utils.py b/litellm/litellm_core_utils/cli_token_utils.py index 53289770c1a..af1b918fc2a 100644 --- a/litellm/litellm_core_utils/cli_token_utils.py +++ b/litellm/litellm_core_utils/cli_token_utils.py @@ -13,15 +13,17 @@ This module has no dependencies on proxy code and can be safely imported at the """ import time +from dataclasses import dataclass from pathlib import Path from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias from pydantic import BaseModel, ConfigDict, ValidationError from litellm.litellm_core_utils.cli_keyring import ( SYSTEM_KEYRING, KeyringDisabled, + KeyringDiscardsWrites, KeyringNotInstalled, KeyringUnreachable, SecretErase, @@ -33,7 +35,23 @@ from litellm.litellm_core_utils.cli_keyring import ( SecretVault, SecretWrite, ) -from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json +from litellm.litellm_core_utils.private_json import ( + commit_staged_json, + discard_staged_json, + ensure_private_dir, + stage_private_json, + write_private_json, +) + + +@dataclass(frozen=True, slots=True) +class CredentialNotSaved: + """The credential was minted but no store would keep it, so this machine has none.""" + + detail: str + + +SecretSave: TypeAlias = SecretWrite | CredentialNotSaved class CliTokenRecord(BaseModel): @@ -85,14 +103,24 @@ def load_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> CliTokenRecord | N return _resolve_secret(record, vault) -def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretWrite: - """Store a freshly minted credential. Reports where its secret material ended up, and why""" +def save_cli_token(record: CliTokenRecord, *, vault: SecretVault = SYSTEM_KEYRING) -> SecretSave: + """Store a freshly minted credential. Reports where its secret material ended up, and why. + + The token file is what makes a keychain-backed credential findable again, so a file that will + not be written takes the keychain copy down with it rather than leaving a live credential + stored under a machine that has no record of it. + """ outcome: Final = ( SecretStored() if record.key is None else vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)) ) - _write_token_file(_without_secret(record) if isinstance(outcome, SecretStored) else record) + try: + _write_token_file(_without_secret(record) if isinstance(outcome, SecretStored) else record) + except OSError as error: + if record.key is not None and isinstance(outcome, SecretStored): + vault.erase() + return CredentialNotSaved(str(error)) return outcome @@ -105,24 +133,30 @@ def clear_cli_token(*, vault: SecretVault = SYSTEM_KEYRING) -> SecretErase: def _nothing_left_behind(outcome: SecretErase) -> bool: - """Whether the keychain can be trusted to hold no credential of ours once the file is gone""" + """Whether the keychain can be trusted to hold no credential of ours once the file is gone. + + A keychain that exists but is out of reach right now is never trusted, whatever the token file + looks like: the login that stored a secret there and the logout that cannot remove it are + separate runs, free to differ in whether the keychain was usable at the time. + """ match outcome: case SecretErased(): return True - case SecretStranded(): + case SecretStranded() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites(): return False - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): - return not _secret_lives_in_keychain() + case KeyringNotInstalled(): + return _file_holds_its_own_secret() -def _secret_lives_in_keychain() -> bool: - """Whether the token file is the metadata half of a pair whose secret half went to a keychain. +def _file_holds_its_own_secret() -> bool: + """Whether the stored login keeps its secret in the token file, ruling out a keychain entry. - A file that still carries its own secret rules one out, which keeps `lite logout` quiet on the - machines that never had a keychain to begin with. + Sound only against a missing `keyring` package, the one way to lose the keychain that had to + hold at storage time too, since nothing here can reach a keychain without it. A file whose + secret half is absent went to a keychain by definition, and so rules nothing out. """ record: Final = _read_token_file() - return record is not None and record.key is None and not record.jwt_token + return record is not None and record.key is not None def get_litellm_gateway_api_key( @@ -179,7 +213,7 @@ def is_cli_token_fresh(token_data: CliTokenRecord, buffer_hours: float = 0.1) -> def _read_token_file() -> CliTokenRecord | None: try: raw: Final = Path(get_cli_token_file_path()).read_text() - except OSError: + except (OSError, ValueError): return None try: return CliTokenRecord.model_validate_json(raw) @@ -193,7 +227,7 @@ def _resolve_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecor return _apply_vault_secret(record, blob, vault) case SecretMissing(): return _migrate_file_secret(record, vault) - case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable(): + case KeyringNotInstalled() | KeyringDisabled() | KeyringUnreachable() | KeyringDiscardsWrites(): return record @@ -216,38 +250,46 @@ def _apply_vault_secret(record: CliTokenRecord, blob: str, vault: SecretVault) - def _migrate_file_secret(record: CliTokenRecord, vault: SecretVault) -> CliTokenRecord | None: - """Move a file-held secret into the vault, but only if the file's copy can be taken away. + """Move a file-held secret into the vault, but only once the file's copy can be taken away. - Migrating without scrubbing would leave the credential live in two stores instead of one, so a - file that will not give its copy up rolls the vault write back rather than widening exposure. + The scrubbed file is staged first so a directory that will not accept it stops the migration + before the keychain is handed anything. Copying the credential into a second store and only + then discovering the first one cannot be cleaned would widen exposure instead of narrowing it, + which is the opposite of what moving it into the keychain is for. """ if record.key is None: return None - if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): + staged: Final = _stage_scrubbed_file(record) + if staged is None: return record - if not _scrub_file_secret(record): + if not isinstance(vault.write(_encode_secret(record.base_url, record.key, record.jwt_token)), SecretStored): + discard_staged_json(staged) + return record + if not _commit_scrubbed_file(staged): vault.erase() return record def _scrub_file_secret(record: CliTokenRecord) -> bool: - """Leave no secret material in the token file once the vault holds it. - - A file that cannot be rewritten without the secret is removed instead. Signing in again costs - the user one command; a live credential left behind in cleartext costs them the credential. - """ + """Leave no secret material in the token file once the vault holds it""" if record.key is None and not record.jwt_token: return True + staged: Final = _stage_scrubbed_file(record) + return staged is not None and _commit_scrubbed_file(staged) + + +def _stage_scrubbed_file(record: CliTokenRecord) -> str | None: + path: Final = Path(get_cli_token_file_path()) try: - _write_token_file(_without_secret(record)) + ensure_private_dir(path.parent) + return stage_private_json(str(path), _without_secret(record).model_dump(exclude_none=True)) except OSError: - return _discard_token_file() - return True + return None -def _discard_token_file() -> bool: +def _commit_scrubbed_file(staged: str) -> bool: try: - Path(get_cli_token_file_path()).unlink(missing_ok=True) + commit_staged_json(staged, get_cli_token_file_path()) except OSError: return False return True diff --git a/litellm/litellm_core_utils/private_json.py b/litellm/litellm_core_utils/private_json.py index 32bc2e169e2..fbeb74aab5a 100644 --- a/litellm/litellm_core_utils/private_json.py +++ b/litellm/litellm_core_utils/private_json.py @@ -16,8 +16,12 @@ def ensure_private_dir(directory: Path) -> None: directory.chmod(PRIVATE_DIR_MODE) -def write_private_json(path: str, data: Mapping[str, object]) -> None: - """Atomically write JSON to path with owner-only permissions (0600)""" +def stage_private_json(path: str, data: Mapping[str, object]) -> str: + """Write JSON to a private temp file beside `path`, ready for `commit_staged_json`. + + Staging is the half that can fail on a read-only or full directory, so callers with something + to lose can find that out before they act on the assumption that the rewrite will land. + """ parent: Final = Path(path).parent parent.mkdir(parents=True, exist_ok=True) fd, tmp_path = tempfile.mkstemp(dir=str(parent), prefix=".tmp-", suffix=".json") @@ -26,6 +30,26 @@ def write_private_json(path: str, data: Mapping[str, object]) -> None: json.dump(data, f, indent=2) f.flush() os.fsync(f.fileno()) - os.replace(tmp_path, path) - finally: + except BaseException: Path(tmp_path).unlink(missing_ok=True) + raise + return tmp_path + + +def commit_staged_json(staged: str, path: str) -> None: + """Move a staged file into place, replacing whatever is there in one step""" + try: + os.replace(staged, path) + except OSError: + Path(staged).unlink(missing_ok=True) + raise + + +def discard_staged_json(staged: str) -> None: + """Throw a staged file away when the change it was part of is abandoned""" + Path(staged).unlink(missing_ok=True) + + +def write_private_json(path: str, data: Mapping[str, object]) -> None: + """Atomically write JSON to path with owner-only permissions (0600)""" + commit_staged_json(stage_private_json(path, data), path) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 4cf18435473..eba9994f7ec 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -15,16 +15,20 @@ from litellm.litellm_core_utils.cli_keyring import ( DISABLE_KEYRING_ENV_VAR, SYSTEM_KEYRING, KeyringDisabled, + KeyringDiscardsWrites, KeyringNotInstalled, KeyringUnreachable, SecretErased, + SecretFound, + SecretMissing, SecretStored, SecretStranded, SecretVault, - SecretWrite, ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, + CredentialNotSaved, + SecretSave, clear_cli_token, get_cli_token_file_path, get_litellm_gateway_api_key, @@ -84,17 +88,19 @@ class CliAuthResult(TypedDict): KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'" +KEYRING_ENABLE_HINT: Final = "keyring --enable (or unset PYTHON_KEYRING_BACKEND)" + STRANDED_CREDENTIAL_MESSAGE: Final = ( "Logged out locally, but your credential is still in the OS keychain and could not be removed." ) -KEYCHAIN_UNREACHABLE_MESSAGE: Final = ( - "Your credential is stored in your OS keychain, which could not be read. Unlock it, or install " - f"the keyring package with: {KEYRING_INSTALL_HINT}. Run 'lite login' to start over." +UNCHECKED_KEYCHAIN_MESSAGE: Final = ( + "Logged out locally, but your OS keychain could not be checked, so a credential stored there by " + "an earlier login may still be usable." ) -def storage_notice(outcome: SecretWrite) -> str: +def storage_notice(outcome: SecretSave) -> str: """Tell the user where the credential ended up, and how to get keychain storage if it did not.""" path: Final = get_cli_token_file_path() match outcome: @@ -109,6 +115,38 @@ def storage_notice(outcome: SecretWrite) -> str: return f"Keychain storage is off ({DISABLE_KEYRING_ENV_VAR}). Credential stored in {path} (owner-only)." case KeyringUnreachable(): return f"No OS keychain available. Credential stored in {path} (owner-only)." + case KeyringDiscardsWrites(): + return ( + f"Your keyring backend keeps nothing it is given, so the credential was stored in {path} " + f"(owner-only) instead. For OS keychain storage, run: {KEYRING_ENABLE_HINT}" + ) + case CredentialNotSaved(detail=detail): + return ( + f"Signed in, but the credential could not be saved to {path}: {detail}. " + "Nothing was kept, so run 'lite login' again once that path is writable." + ) + + +def keychain_unreadable_notice(vault: SecretVault) -> str: + """Explain why the secret half of a stored login cannot be produced, and what fixes it""" + match vault.read(): + case KeyringNotInstalled(): + return ( + "Your credential is in your OS keychain, which this install cannot read without the " + f"keyring package. Install it with: {KEYRING_INSTALL_HINT}, or run 'lite login' to start over." + ) + case KeyringDisabled(): + return ( + f"Your credential is in your OS keychain, which {DISABLE_KEYRING_ENV_VAR} is blocking. " + "Unset it, or run 'lite login' to start over." + ) + case KeyringUnreachable() | KeyringDiscardsWrites(): + return ( + "Your credential is in your OS keychain, which could not be read. Unlock it, or run " + "'lite login' to start over." + ) + case SecretFound() | SecretMissing(): + return "Your credential could not be read from your OS keychain. Run 'lite login' to start over." def context_secret_vault(ctx: click.Context) -> SecretVault: @@ -715,6 +753,8 @@ def login(ctx: click.Context, config_claude: bool): click.echo("\nLogin successful!") click.echo(f"JWT Token: {api_key[:20]}...") click.echo(storage_notice(stored)) + if isinstance(stored, CredentialNotSaved): + return click.echo("You can now use the CLI without specifying --api-key") if config_claude: @@ -751,14 +791,17 @@ def logout(ctx: click.Context): match clear_cli_token(vault=context_secret_vault(ctx)): case SecretErased(): click.echo("Logged out successfully. Authentication token cleared.") + case SecretStranded(): + click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo("Unlock your keychain and run 'lite logout' again to clear it.") case KeyringNotInstalled(): click.echo(STRANDED_CREDENTIAL_MESSAGE) click.echo(f"Install the keyring package with: {KEYRING_INSTALL_HINT}, then run 'lite logout' again.") case KeyringDisabled(): - click.echo(STRANDED_CREDENTIAL_MESSAGE) + click.echo(UNCHECKED_KEYCHAIN_MESSAGE) click.echo(f"Unset {DISABLE_KEYRING_ENV_VAR} and run 'lite logout' again to clear it.") - case SecretStranded() | KeyringUnreachable(): - click.echo(STRANDED_CREDENTIAL_MESSAGE) + case KeyringUnreachable() | KeyringDiscardsWrites(): + click.echo(UNCHECKED_KEYCHAIN_MESSAGE) click.echo("Unlock your keychain and run 'lite logout' again to clear it.") @@ -795,7 +838,7 @@ def print_token(ctx: click.Context): api_key: Final = token_data.key if not api_key: - click.echo(KEYCHAIN_UNREACHABLE_MESSAGE, err=True) + click.echo(keychain_unreadable_notice(context_secret_vault(ctx)), err=True) sys.exit(1) click.echo(api_key) @@ -821,7 +864,7 @@ def whoami(ctx: click.Context): click.echo(f"Token age: {age_hours:.1f} hours") if token_data.key is None: - click.echo(KEYCHAIN_UNREACHABLE_MESSAGE) + click.echo(keychain_unreadable_notice(context_secret_vault(ctx))) if age_hours > CLI_JWT_EXPIRATION_HOURS: click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.") diff --git a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py index cbc93bbde71..e0f5f99dc1b 100644 --- a/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_cli_token_utils.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.cli_keyring import ( SecretFound, SecretMissing, KeyringDisabled, + KeyringDiscardsWrites, KeyringNotInstalled, KeyringUnreachable, SecretErased, @@ -23,6 +24,7 @@ from litellm.litellm_core_utils.cli_keyring import ( ) from litellm.litellm_core_utils.cli_token_utils import ( CliTokenRecord, + CredentialNotSaved, clear_cli_token, get_cli_token_file_path, get_litellm_gateway_api_key, @@ -225,6 +227,15 @@ class TestLoadCliToken: assert record.key == "sk-legacy" + def test_a_token_file_that_is_not_text_is_not_a_login(self, isolated_home, secret_vault_factory): + """A truncated write or a half-synced backup can leave bytes that are not UTF-8 at all. + Reading them must fail the way an absent file does, not crash every `lite` command.""" + path = _token_file(isolated_home) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\xff\xfe not utf-8 at all") + + assert load_cli_token(vault=secret_vault_factory()) is None + def test_corrupt_token_file_is_not_a_login(self, isolated_home, secret_vault_factory): _token_file(isolated_home).parent.mkdir() _token_file(isolated_home).write_text("not json at all {{{") @@ -301,6 +312,40 @@ class TestSaveCliToken: assert stat.S_IMODE(config_dir.stat().st_mode) == 0o700 + def test_a_credential_no_store_would_keep_is_reported_rather_than_raised( + self, isolated_home, secret_vault_factory, monkeypatch + ): + """`lite login` catches whatever escapes here and calls it an authentication failure, which + is the one thing that did not happen: the proxy minted a real credential. Saying so lets the + user act on the actual problem instead of retrying a sign-in that already worked.""" + + def _explode(*args, **kwargs): + raise OSError("read-only file system") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + + outcome = save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=secret_vault_factory()) + + assert isinstance(outcome, CredentialNotSaved) + assert "read-only file system" in outcome.detail + + def test_a_credential_the_file_will_not_record_is_taken_back_out_of_the_keychain( + self, isolated_home, secret_vault_factory, monkeypatch + ): + """The token file is what makes a keychain entry findable again. Leaving the secret in the + keychain with nothing pointing at it strands a live credential under a machine that has no + idea it is there, and no `lite logout` would ever go looking for it.""" + vault = secret_vault_factory() + + def _explode(*args, **kwargs): + raise OSError("read-only file system") + + monkeypatch.setattr("litellm.litellm_core_utils.private_json.json.dump", _explode) + + save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-new"), vault=vault) + + assert vault.blob is None + def test_a_failed_write_leaves_the_previous_credential_intact(self, isolated_home, secret_vault_factory, monkeypatch): path = _write_legacy_file(isolated_home) before = path.read_text() @@ -340,9 +385,11 @@ class TestScrubFailure: assert json.loads(path.read_text())["key"] == "sk-legacy" assert vault.blob is None - def test_a_file_that_cannot_be_rewritten_is_removed_instead( + def test_a_full_disk_stops_the_migration_before_the_keychain_is_handed_anything( self, isolated_home, secret_vault_factory, monkeypatch ): + """The scrubbed file is staged first precisely so this is knowable in advance. A disk that + cannot take the rewrite leaves the credential where it already was, in one store.""" path = _write_legacy_file(isolated_home) vault = secret_vault_factory() @@ -354,8 +401,8 @@ class TestScrubFailure: record = load_cli_token(vault=vault) assert record.key == "sk-legacy" - assert json.loads(vault.blob)["key"] == "sk-legacy" - assert not path.exists() + assert vault.blob is None + assert json.loads(path.read_text())["key"] == "sk-legacy" assert list(path.parent.glob(".tmp-*")) == [] @@ -376,12 +423,39 @@ class TestClearCliToken: assert clear_cli_token(vault=vault) == SecretStranded() assert not _token_file(isolated_home).exists() + @pytest.mark.parametrize( + "failure", [KeyringDisabled(), KeyringUnreachable(), KeyringDiscardsWrites()] + ) + def test_a_secret_in_the_file_is_no_evidence_about_a_keychain_that_exists( + self, isolated_home, secret_vault_factory, failure + ): + """Store a secret in the keychain, sign in again while the keychain is unusable so the new + secret lands in the file, then log out while it is still unusable. The file now carries its + own secret and the first login's entry is still there, so reading the file as proof of a + clean keychain reports a logout that did not happen.""" + _write_legacy_file(isolated_home) + vault = secret_vault_factory(available=False, failure=failure) + + assert clear_cli_token(vault=vault) == failure + assert not _token_file(isolated_home).exists() + + def test_a_second_logout_still_reports_the_keychain_it_could_not_clear( + self, isolated_home, secret_vault_factory + ): + """The first logout deletes the file and tells the user to run it again once the keychain is + reachable. If the second run reads that missing file as proof of a clean keychain, the advice + turns into the very false all-clear it was issued to prevent.""" + _write_metadata_only_file(isolated_home) + vault = secret_vault_factory(available=False, failure=KeyringUnreachable()) + + assert clear_cli_token(vault=vault) == KeyringUnreachable() + assert clear_cli_token(vault=vault) == KeyringUnreachable() + def test_logout_from_an_install_without_keyring_does_not_claim_the_keychain_is_clear( self, isolated_home, secret_vault_factory ): - """Log in where `litellm[cli]` is installed and the secret goes to the OS keychain; log out - from a venv without it and the entry survives, because it belongs to the OS rather than to - the package. Reporting a clean logout there leaves a live credential the user thinks is gone.""" + """A file holding only metadata put its secret in a keychain by definition. Losing the + package that reaches it does not take the entry with it, so this cannot report success.""" _write_metadata_only_file(isolated_home) vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) @@ -389,8 +463,9 @@ class TestClearCliToken: assert not _token_file(isolated_home).exists() def test_a_file_backed_login_logs_out_quietly_without_keyring(self, isolated_home, secret_vault_factory): - """The complement: a user who never had a keychain keeps their whole credential in the file, - so removing it is a complete logout and must not warn about an entry that cannot exist.""" + """The complement, and the one inference the file does support: nothing here can reach a + keychain without the package, so an install that lacks it and a file that still holds its + own secret between them account for the whole credential.""" _write_legacy_file(isolated_home) vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) @@ -417,11 +492,12 @@ class TestIsCliTokenFresh: class _FakeKeyringModule: - def __init__(self, stored=None, *, get_error=None, set_error=None, delete_error=None): + def __init__(self, stored=None, *, get_error=None, set_error=None, delete_error=None, discard=False): self.stored = stored self.get_error = get_error self.set_error = set_error self.delete_error = delete_error + self.discard = discard self.calls = [] def get_password(self, service_name, username): @@ -434,6 +510,8 @@ class _FakeKeyringModule: self.calls.append(("set", service_name, username)) if self.set_error is not None: raise self.set_error + if self.discard: + return self.stored = password def delete_password(self, service_name, username): @@ -503,6 +581,41 @@ class TestKeyringVault: assert KeyringVault().erase() == SecretStranded() + def test_a_backend_that_keeps_nothing_is_not_a_successful_write(self, install_fake_keyring): + """keyring's null backend accepts every write, stores nothing, and raises nothing to say so. + Taking its silence for success is how a credential gets deleted: the caller drops its own + copy on our word. Only reading the value back tells the two apart.""" + fake = install_fake_keyring(_FakeKeyringModule(discard=True)) + + assert KeyringVault().write("blob-1") == KeyringDiscardsWrites() + assert fake.stored is None + + def test_the_real_null_backend_is_rejected(self, monkeypatch): + """Pinned against the actual library rather than the double above, because the whole risk is + that upstream's no-op write looks exactly like a successful one.""" + keyring = pytest.importorskip("keyring") + null_backend = pytest.importorskip("keyring.backends.null") + monkeypatch.delenv(DISABLE_KEYRING_ENV_VAR, raising=False) + previous = keyring.get_keyring() + keyring.set_keyring(null_backend.Keyring()) + try: + assert KeyringVault().write("blob-1") == KeyringDiscardsWrites() + finally: + keyring.set_keyring(previous) + + def test_a_credential_survives_a_backend_that_keeps_nothing( + self, isolated_home, install_fake_keyring + ): + """The end of the same story: the credential must still be usable afterwards. Reporting the + discard is only worth anything if the token file then keeps the copy the keychain refused.""" + install_fake_keyring(_FakeKeyringModule(discard=True)) + + outcome = save_cli_token(CliTokenRecord(base_url=SERVER, key="sk-only-copy")) + + assert outcome == KeyringDiscardsWrites() + assert json.loads(_token_file(isolated_home).read_text())["key"] == "sk-only-copy" + assert load_cli_token().key == "sk-only-copy" + def test_erasing_a_locked_keychain_is_a_failure(self, install_fake_keyring): install_fake_keyring(_FakeKeyringModule(get_error=RuntimeError("locked"))) diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 33bd8307c21..8e1551c0720 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -16,12 +16,13 @@ from litellm.constants import CLI_JWT_EXPIRATION_HOURS from litellm.litellm_core_utils.cli_keyring import ( DISABLE_KEYRING_ENV_VAR, KeyringDisabled, + KeyringDiscardsWrites, KeyringNotInstalled, ) from litellm.litellm_core_utils.cli_token_utils import CliTokenRecord, save_cli_token from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.auth import ( - KEYCHAIN_UNREACHABLE_MESSAGE, + DISABLE_KEYRING_ENV_VAR, get_stored_api_key, login, logout, @@ -461,6 +462,20 @@ class TestLogoutCommand: assert "still in the OS keychain" in result.output assert "pip install 'litellm[cli]'" in result.output + def test_logout_does_not_call_an_unusable_keychain_clean(self, isolated_home, secret_vault_factory): + """A keychain-backed login, then a login that fell back to the file because the keychain had + become unusable, leaves the first entry live. The file's own secret says nothing about it, + so a clean bill of health here is the one answer that cannot be justified.""" + _write_token_file(isolated_home, key="sk-in-file") + vault = secret_vault_factory(available=False, failure=KeyringDisabled()) + + result = self.runner.invoke(logout, obj={"secret_vault": vault}) + + assert result.exit_code == 0 + assert "Logged out successfully" not in result.output + assert "could not be checked" in result.output + assert DISABLE_KEYRING_ENV_VAR in result.output + def test_logout_warns_when_the_keychain_refuses_to_release_the_entry( self, isolated_home, secret_vault_factory ): @@ -1003,6 +1018,19 @@ class TestKeychainBackedCommands: assert "No OS keychain available" not in result.output assert json.loads(token_file.read_text())["key"] == "sk-minted" + def test_login_keeps_the_credential_when_the_backend_keeps_nothing( + self, isolated_home, secret_vault_factory + ): + """A backend that accepts writes and stores nothing must not be reported as keychain + storage, because the file is then told to drop the only remaining copy.""" + result = self._login(secret_vault_factory(available=False, failure=KeyringDiscardsWrites())) + + token_file = isolated_home / ".litellm" / "token.json" + assert result.exit_code == 0 + assert "Credential stored in your OS keychain." not in result.output + assert "keyring --enable" in result.output + assert json.loads(token_file.read_text())["key"] == "sk-minted" + def test_login_names_the_kill_switch_instead_of_blaming_the_machine( self, isolated_home, secret_vault_factory ): @@ -1061,7 +1089,8 @@ class TestKeychainBackedCommands: result = self.runner.invoke(print_token, obj=obj) assert result.exit_code == 1 - assert KEYCHAIN_UNREACHABLE_MESSAGE in result.output + assert "could not be read" in result.output + assert "lite login" in result.output def test_whoami_flags_a_locked_keychain(self, isolated_home, secret_vault_factory): _write_home_json( @@ -1074,7 +1103,34 @@ class TestKeychainBackedCommands: result = self.runner.invoke(whoami, obj=obj) assert "Authenticated" in result.output - assert KEYCHAIN_UNREACHABLE_MESSAGE in result.output + assert "could not be read" in result.output + + def test_whoami_names_the_kill_switch_rather_than_a_missing_package( + self, isolated_home, secret_vault_factory + ): + """Every unreachable keychain used to be described as a locked one needing the keyring + package installed. Someone who set the kill switch has the package and an unlocked keychain, + so that advice sends them to fix two things that were never wrong.""" + _write_token_file(isolated_home, key=None) + vault = secret_vault_factory(available=False, failure=KeyringDisabled()) + + result = self.runner.invoke(whoami, obj={"base_url": "https://test.example.com", "secret_vault": vault}) + + assert DISABLE_KEYRING_ENV_VAR in result.output + assert "pip install" not in result.output + + def test_print_token_points_an_install_without_keyring_at_the_package( + self, isolated_home, secret_vault_factory + ): + _write_token_file(isolated_home, key=None) + vault = secret_vault_factory(available=False, failure=KeyringNotInstalled()) + obj = {"base_url": "https://test.example.com", "secret_vault": vault} + + result = self.runner.invoke(print_token, obj=obj) + + assert result.exit_code == 1 + assert "pip install 'litellm[cli]'" in result.output + assert DISABLE_KEYRING_ENV_VAR not in result.output class TestApiKeyPrecedence: From 4829bb3a151c4b4e58580313784862eb966921e1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:25:06 -0700 Subject: [PATCH 027/226] fix(prompt_management): don't route requests without prompt_id to prompt managers that can't run them UI-injected empty vector_store_ids/tags/guardrails on a DB model tripped the dynamic-param check, and the prompt-management fallback then handed the request to the first registered prompt manager (e.g. a saved dotprompt), whose sync path raised "prompt_id is required" as a 500 on every /chat/completions call. Empty dynamic params no longer count as a trigger, the fallback skips managers whose should_run_prompt_management declines a None prompt_id, and the sync base path returns the request unchanged for a None prompt_id like the async path. --- .../integrations/prompt_management_base.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 30 ++++++- .../test_litellm_logging.py | 78 +++++++++++++++++++ 3 files changed, 105 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index d16afa92ec2..81c01599e77 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -165,7 +165,7 @@ class PromptManagementBase(ABC): ignore_prompt_manager_optional_params: bool | None = False, ) -> tuple[str, list[AllMessageValues], dict]: if prompt_id is None: - raise ValueError("prompt_id is required for Prompt Management Base class") + return model, messages, non_default_params if not self.should_run_prompt_management( prompt_id=prompt_id, prompt_spec=prompt_spec, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 91a312b4f45..9b7707eabe1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -832,8 +832,8 @@ class Logging(LiteLLMLoggingBaseClass): eg. AnthropicCacheControlHook and BedrockKnowledgeBaseHook both don't require a `prompt_id` to be passed in, they are triggered by dynamic params """ - for param in non_default_params: - if param in DynamicPromptManagementParamLiteral.list_all_params(): + for param in DynamicPromptManagementParamLiteral.list_all_params(): + if non_default_params.get(param): return True ############################################################################# @@ -966,6 +966,23 @@ class Logging(LiteLLMLoggingBaseClass): return None + @staticmethod + def _prompt_manager_runs_without_prompt_id( + logger: CustomLogger, + prompt_spec: PromptSpec | None, + dynamic_callback_params: StandardCallbackDynamicParams | None, + ) -> bool: + if not isinstance(logger, CustomPromptManagement): + return False + try: + return logger.should_run_prompt_management( + prompt_id=None, + prompt_spec=prompt_spec, + dynamic_callback_params=dynamic_callback_params or StandardCallbackDynamicParams(), + ) + except Exception: + return False + def get_custom_logger_for_prompt_management( self, model: str, @@ -1016,8 +1033,13 @@ class Logging(LiteLLMLoggingBaseClass): callback_type=CustomPromptManagement ) - if prompt_management_loggers: - logger: Final = prompt_management_loggers[0] + for logger in prompt_management_loggers: + if prompt_id is None and not self._prompt_manager_runs_without_prompt_id( + logger=logger, + prompt_spec=prompt_spec, + dynamic_callback_params=dynamic_callback_params, + ): + continue self.model_call_details["prompt_integration"] = logger.__class__.__name__ return logger diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index c2d73ea467d..05aa65034fb 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -5105,3 +5105,81 @@ def test_set_cost_breakdown_stores_vertex_location(): cost_for_built_in_tools_cost_usd_dollar=0.0, ) assert no_location.cost_breakdown.get("vertex_location") is None + + +def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_path, monkeypatch): + """ + Regression for UI-injected `vector_store_ids: []` and always-on non-empty `vector_store_ids` + with a registered prompt manager (e.g. dotprompt): requests without a prompt_id 500'd with + "prompt_id is required for Prompt Management Base class" instead of completing normally. + """ + from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + VectorStorePreCallHook, + ) + from litellm.types.vector_stores import LiteLLM_ManagedVectorStore + from litellm.vector_stores.vector_store_registry import VectorStoreRegistry + + (tmp_path / "stem.prompt").write_text("---\nmodel: gemini-2.5-flash\n---\nyou are a stem tutor\n") + dotprompt_manager = DotpromptManager(prompt_directory=str(tmp_path)) + litellm.logging_callback_manager.add_litellm_callback(dotprompt_manager) + monkeypatch.setattr( + litellm, + "vector_store_registry", + VectorStoreRegistry( + vector_stores=[LiteLLM_ManagedVectorStore(vector_store_id="vs_123", custom_llm_provider="openai")] + ), + ) + + messages = [{"role": "user", "content": "hi"}] + try: + assert not logging_obj.should_run_prompt_management_hooks( + prompt_id=None, non_default_params={"vector_store_ids": []} + ) + + assert logging_obj.get_chat_completion_prompt( + model="gemini-2.5-flash", + messages=messages, + non_default_params={"vector_store_ids": []}, + prompt_variables=None, + prompt_id=None, + ) == ("gemini-2.5-flash", messages, {"vector_store_ids": []}) + + assert dotprompt_manager.get_chat_completion_prompt( + model="gemini-2.5-flash", + messages=messages, + non_default_params={}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) == ("gemini-2.5-flash", messages, {}) + + assert logging_obj.should_run_prompt_management_hooks( + prompt_id=None, non_default_params={"vector_store_ids": ["vs_123"]} + ) + assert isinstance( + logging_obj.get_custom_logger_for_prompt_management( + model="gemini-2.5-flash", + non_default_params={"vector_store_ids": ["vs_123"]}, + prompt_id=None, + dynamic_callback_params={}, + ), + VectorStorePreCallHook, + ) + + assert isinstance( + logging_obj.get_custom_logger_for_prompt_management( + model="gemini-2.5-flash", + non_default_params={}, + prompt_id="stem", + dynamic_callback_params={}, + ), + DotpromptManager, + ) + finally: + litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, dotprompt_manager) + litellm.logging_callback_manager.remove_callback_from_list_by_object( + litellm._async_success_callback, dotprompt_manager + ) + for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]: + litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook) From b780b1e23c160ee8b8eb3effe471c280547fa10f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:38:15 -0700 Subject: [PATCH 028/226] fix(arize): decline prompt management runs without a prompt_id Arize Phoenix claimed it could run without a prompt_id while its compiler requires one, so the no-prompt_id fallback could select it and fail instead of reaching the vector-store hook. It now declines like the other managers. --- .../arize/arize_phoenix_prompt_manager.py | 6 +++--- .../litellm_core_utils/test_litellm_logging.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/arize/arize_phoenix_prompt_manager.py b/litellm/integrations/arize/arize_phoenix_prompt_manager.py index fa178a02752..71f4902bbe5 100644 --- a/litellm/integrations/arize/arize_phoenix_prompt_manager.py +++ b/litellm/integrations/arize/arize_phoenix_prompt_manager.py @@ -359,10 +359,10 @@ class ArizePhoenixPromptManager(CustomPromptManagement): """ Determine if prompt management should run based on the prompt_id. - For Arize Phoenix, we always return True and handle the prompt loading - in the _compile_prompt_helper method. + Arize Phoenix needs a prompt_id to compile, so it declines requests without one; + prompt loading itself happens in the _compile_prompt_helper method. """ - return True + return prompt_id is not None def _compile_prompt_helper( self, diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 05aa65034fb..db64340f895 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -5113,6 +5113,7 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa with a registered prompt manager (e.g. dotprompt): requests without a prompt_id 500'd with "prompt_id is required for Prompt Management Base class" instead of completing normally. """ + from litellm.integrations.arize.arize_phoenix_prompt_manager import ArizePhoenixPromptManager from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( VectorStorePreCallHook, @@ -5122,7 +5123,9 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa (tmp_path / "stem.prompt").write_text("---\nmodel: gemini-2.5-flash\n---\nyou are a stem tutor\n") dotprompt_manager = DotpromptManager(prompt_directory=str(tmp_path)) + arize_manager = ArizePhoenixPromptManager(api_key="fake-key", api_base="http://127.0.0.1:9") litellm.logging_callback_manager.add_litellm_callback(dotprompt_manager) + litellm.logging_callback_manager.add_litellm_callback(arize_manager) monkeypatch.setattr( litellm, "vector_store_registry", @@ -5154,6 +5157,10 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa dynamic_callback_params={}, ) == ("gemini-2.5-flash", messages, {}) + assert not arize_manager.should_run_prompt_management( + prompt_id=None, prompt_spec=None, dynamic_callback_params={} + ) + assert logging_obj.should_run_prompt_management_hooks( prompt_id=None, non_default_params={"vector_store_ids": ["vs_123"]} ) @@ -5177,9 +5184,10 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa DotpromptManager, ) finally: - litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, dotprompt_manager) - litellm.logging_callback_manager.remove_callback_from_list_by_object( - litellm._async_success_callback, dotprompt_manager - ) + for manager in (dotprompt_manager, arize_manager): + litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, manager) + litellm.logging_callback_manager.remove_callback_from_list_by_object( + litellm._async_success_callback, manager + ) for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]: litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook) From 93c14610748bb724702f2f4616bc8137841c080e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 19 Aug 2026 22:52:36 -0700 Subject: [PATCH 029/226] fix(ui): restore hover feedback and dark-mode variants lost in the token migration (#37579) * fix(ui): restore hover feedback and dark-mode variants lost in the token migration PR #37576 mapped hardcoded Tailwind palette classes onto semantic tokens. Two-tone hover pairs collapsed onto a single token, so 116 hover utilities across 49 files became identical to their base class and produced no visible feedback, and in seven files a dark: variant was dropped while its hardcoded light partner survived, leaving those elements stuck light in dark mode. Hover states now follow the alpha-step idiom the shadcn primitives already use (hover:bg-primary/80, hover:bg-success/20): a duplicated hover:text-X or hover:bg-X becomes /80, hover:border-border becomes hover:border-ring, and a duplicate is dropped where another hover utility on the element already carries the change. One transition-colors that no longer animated anything is removed. For the dark-mode gaps, indigo maps onto info and amber onto warning. There is no purple token in globals.css, so the purple sites keep their palette classes and get their dark: partner back. * fix(ui): add an eslint rule that fails a hover: utility identical to its base The token migration collapsed two-tone hover pairs by hand, so nothing catches the next one. `local/no-noop-hover-variant` reads every string literal and template chunk and errors when a `hover:X` sits alongside a bare `X`, which is exactly the shape that renders no hover feedback. It ships at error with no suppression baseline, so the eleven sites that already carried a dead hover before the migration are fixed here too. The rule reads one class string at a time, so a base class supplied by a different ternary branch than its hover partner is left alone: a selected row whose resting colour already matches its hover colour is deliberate, not a bug. --- ui/litellm-dashboard/eslint.config.mjs | 5 ++ .../scripts/eslint-rules/index.mjs | 2 + .../eslint-rules/no-noop-hover-variant.mjs | 40 +++++++++++++++ .../agents/_components/add_agent_form.tsx | 12 ++--- .../agents/_components/agent_form_fields.tsx | 4 +- .../_components/provider_discount_table.tsx | 4 +- .../_components/provider_margin_table.tsx | 4 +- .../_components/TeamGuardrailsTab.tsx | 18 +++---- .../_components/add_guardrail_form.tsx | 10 ++-- .../_components/guardrail_optional_params.tsx | 2 +- .../_components/llm_judge/LLMJudgeFields.tsx | 2 +- .../_components/AwsSigV4Fields.tsx | 4 +- .../_components/CreateMCPServer.tsx | 8 +-- .../_components/DcrBridgeToggle.tsx | 2 +- .../_components/EnvVarsSection.tsx | 2 +- .../_components/IdJagFormFields.tsx | 2 +- .../_components/MCPPermissionManagement.tsx | 14 +++--- .../_components/MCPSubmissionsTab.tsx | 12 ++--- .../_components/MCPToolsetsTab.tsx | 28 ++++++++--- .../_components/OAuthFormFields.tsx | 4 +- .../_components/OpenAPIFormSection.tsx | 2 +- .../_components/OpenApiByokFields.tsx | 6 +-- .../_components/StdioConfiguration.tsx | 2 +- .../TokenEndpointAuthMethodField.tsx | 2 +- .../_components/TokenExchangeFormFields.tsx | 2 +- .../mcp-servers/_components/ToolTestPanel.tsx | 8 +-- .../mcp-servers/_components/mcp_connect.tsx | 20 ++++---- .../_components/mcp_server_edit.tsx | 22 ++++---- .../components/chat_ui/A2AMetrics.tsx | 2 +- .../chat_ui/AdditionalModelSettings.tsx | 2 +- .../components/chat_ui/AgentBuilderView.tsx | 2 +- .../playground/components/chat_ui/ChatUI.tsx | 10 ++-- .../chat_ui/CodeInterpreterOutput.tsx | 2 +- .../chat_ui/CodeInterpreterTool.tsx | 2 +- .../components/complianceUI/ComplianceUI.tsx | 16 +++--- .../policies/_components/add_policy_form.tsx | 6 +-- .../_components/ai_suggestion_modal.tsx | 4 +- .../_components/components/CreateTagModal.tsx | 2 +- .../tag-management/_components/tag_info.tsx | 2 +- .../components/EntityUsage/EntityUsage.tsx | 2 +- .../_components/view_users/user_info_view.tsx | 2 +- ui/litellm-dashboard/src/app/chat/page.tsx | 4 +- .../AIHub/UsefulLinksManagement.tsx | 4 +- .../src/components/HelpLink.tsx | 4 +- .../components/Navbar/navProductLinkClass.ts | 2 +- .../SidebarAccountMenu/SidebarAccountMenu.tsx | 2 +- .../src/components/SidebarUsageCard.tsx | 2 +- .../ToolPolicies/ToolPoliciesPanel.tsx | 2 +- .../components/add_model/KeywordTierRules.tsx | 2 +- .../components/bulk_create_users_button.tsx | 2 +- .../PassThroughGuardrailsSection.tsx | 2 +- .../key_team_helpers/BudgetWindowsEditor.tsx | 2 +- .../MCPToolPermissions.tsx | 4 +- .../mcp_tools/ByokCredentialModal.tsx | 4 +- .../HealthChecksTableColumns.tsx | 4 +- .../src/components/model_filters.tsx | 5 +- .../components/model_group_alias_settings.tsx | 2 +- .../organisms/create_key_button.tsx | 2 +- .../permissions/MCPServerPermissions.tsx | 2 +- .../src/components/public_model_hub.tsx | 14 +++--- .../router_settings/TagFilteringToggle.tsx | 2 +- .../components/shared/DataTable/DataTable.tsx | 2 +- .../shared/advanced_date_picker.tsx | 6 +-- .../src/components/team/LoggingSettings.tsx | 2 +- .../components/view_logs/LogsTableToolbar.tsx | 2 +- .../no-noop-hover-variant.test.ts | 50 +++++++++++++++++++ 66 files changed, 269 insertions(+), 157 deletions(-) create mode 100644 ui/litellm-dashboard/scripts/eslint-rules/no-noop-hover-variant.mjs create mode 100644 ui/litellm-dashboard/tests/eslint-rules/no-noop-hover-variant.test.ts diff --git a/ui/litellm-dashboard/eslint.config.mjs b/ui/litellm-dashboard/eslint.config.mjs index df10ca1befa..6d7ca2ad071 100644 --- a/ui/litellm-dashboard/eslint.config.mjs +++ b/ui/litellm-dashboard/eslint.config.mjs @@ -22,6 +22,7 @@ const eslintConfig = [ "local/no-large-inline-object-arg": "warn", "local/no-long-condition-chain": "warn", "local/no-complex-jsx-arrow": ["error", { maxStatements: 2 }], + "local/no-noop-hover-variant": "error", "@typescript-eslint/no-explicit-any": "warn", "no-console": ["warn", { allow: ["warn", "error"] }], "@typescript-eslint/no-unused-vars": "off", @@ -81,6 +82,10 @@ const eslintConfig = [ "no-restricted-syntax": "off", }, }, + { + files: ["tests/eslint-rules/**/*.{ts,tsx}"], + rules: { "local/no-noop-hover-variant": "off" }, + }, { files: ["src/**/*.test.{ts,tsx}", "tests/**/*.{ts,tsx}"], plugins: { "testing-library": testingLibrary, "jest-dom": jestDom }, diff --git a/ui/litellm-dashboard/scripts/eslint-rules/index.mjs b/ui/litellm-dashboard/scripts/eslint-rules/index.mjs index 9e9f901a6df..983399ae4a3 100644 --- a/ui/litellm-dashboard/scripts/eslint-rules/index.mjs +++ b/ui/litellm-dashboard/scripts/eslint-rules/index.mjs @@ -2,6 +2,7 @@ import noLargeInlineObjectArg from "./no-large-inline-object-arg.mjs"; import noLongConditionChain from "./no-long-condition-chain.mjs"; import noComplexJsxArrow from "./no-complex-jsx-arrow.mjs"; import filenamePascalCase from "./filename-pascal-case.mjs"; +import noNoopHoverVariant from "./no-noop-hover-variant.mjs"; const plugin = { rules: { @@ -9,6 +10,7 @@ const plugin = { "no-long-condition-chain": noLongConditionChain, "no-complex-jsx-arrow": noComplexJsxArrow, "filename-pascal-case": filenamePascalCase, + "no-noop-hover-variant": noNoopHoverVariant, }, }; diff --git a/ui/litellm-dashboard/scripts/eslint-rules/no-noop-hover-variant.mjs b/ui/litellm-dashboard/scripts/eslint-rules/no-noop-hover-variant.mjs new file mode 100644 index 00000000000..75a5b09596c --- /dev/null +++ b/ui/litellm-dashboard/scripts/eslint-rules/no-noop-hover-variant.mjs @@ -0,0 +1,40 @@ +const noopHovers = (value) => { + const tokens = value.split(/\s+/).filter(Boolean); + const bare = new Set(tokens.filter((t) => !t.includes(":"))); + return tokens + .filter((t) => t.startsWith("hover:")) + .map((t) => [t, t.slice("hover:".length)]) + .filter(([, base]) => bare.has(base)); +}; + +const rule = { + meta: { + type: "problem", + docs: { + description: + "Disallow a hover: utility whose value is identical to the base utility in the same class string, which renders no hover feedback.", + }, + schema: [], + messages: { + noop: "`{{hover}}` is identical to the base `{{base}}`, so hovering changes nothing. Give it a distinct value (e.g. `{{hover}}/80`) or drop it.", + }, + }, + create(context) { + const check = (node, value) => { + if (typeof value !== "string" || !value.includes("hover:")) return; + for (const [hover, base] of noopHovers(value)) { + context.report({ node, messageId: "noop", data: { hover, base } }); + } + }; + return { + Literal(node) { + check(node, node.value); + }, + TemplateElement(node) { + check(node, node.value.cooked); + }, + }; + }, +}; + +export default rule; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx index ed0d6193e5a..b5c04029d69 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx @@ -758,7 +758,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
Not listed?
- + @@ -877,7 +877,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
{/* Agent name chip */}
- + {agentName} @@ -906,7 +906,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
setKeyAssignOption("create_new")} @@ -916,7 +916,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
- + Create a new key for this agent

A dedicated key scoped to this agent.

@@ -943,7 +943,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
setKeyAssignOption("existing_key")} @@ -993,7 +993,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok

Agent Created!

- + {createdAgentName} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_form_fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_form_fields.tsx index 243da085cb7..622e5ea7c5c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_form_fields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_form_fields.tsx @@ -131,7 +131,7 @@ const SkillsFieldArray = () => { @@ -122,7 +122,7 @@ const ProviderDiscountTable: React.FC = ({ size="icon-sm" aria-label={`Edit discount for ${displayName}`} onClick={() => handleStartEdit(row.provider, row.discount)} - className="cursor-pointer text-info hover:text-info" + className="cursor-pointer text-info hover:text-info/80" > diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx index 37db4afc504..fa322d1b4d1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx @@ -152,7 +152,7 @@ const ProviderMarginTable: React.FC = ({ size="icon-sm" aria-label={`Save margin for ${displayName}`} onClick={() => handleSaveEdit(row.provider)} - className="cursor-pointer text-success hover:text-success" + className="cursor-pointer text-success hover:text-success/80" > @@ -174,7 +174,7 @@ const ProviderMarginTable: React.FC = ({ size="icon-sm" aria-label={`Edit margin for ${displayName}`} onClick={() => handleStartEdit(row.provider, row.margin)} - className="cursor-pointer text-info hover:text-info" + className="cursor-pointer text-info hover:text-info/80" > diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx index ea8355bfed0..cc2d2051a24 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx @@ -199,7 +199,7 @@ const STATUS_CONFIG: Record = { - "ML Platform": "bg-purple-100 text-purple-700", + "ML Platform": "bg-purple-100 text-purple-700 dark:bg-purple-900 dark:text-purple-300", "Data Science": "bg-info/15 text-info", Security: "bg-destructive/15 text-destructive", "Customer Success": "bg-warning/15 text-warning", @@ -311,7 +311,7 @@ function GuardrailCard({ return (
@@ -358,7 +358,7 @@ function GuardrailCard({ @@ -600,7 +600,7 @@ function DetailPanel({ setNewStaticHeaderValue(""); } }} - className="text-xs font-medium text-info hover:text-info border border-info/20 bg-info/10 hover:bg-info/15 px-2 py-1.5 rounded-sm transition-colors shrink-0" + className="text-xs font-medium text-info border border-info/20 bg-info/10 hover:bg-info/15 px-2 py-1.5 rounded-sm transition-colors shrink-0" > Add @@ -671,7 +671,7 @@ function DetailPanel({ setNewExtraHeader(""); } }} - className="text-xs font-medium text-info hover:text-info border border-info/20 bg-info/10 hover:bg-info/15 px-2 py-1.5 rounded-sm transition-colors" + className="text-xs font-medium text-info border border-info/20 bg-info/10 hover:bg-info/15 px-2 py-1.5 rounded-sm transition-colors" > Add @@ -682,7 +682,7 @@ function DetailPanel({
- {!isLast &&
} + {!isLast &&
}
{/* Step content */} @@ -1142,7 +1142,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a {step.optional && !isCurrent && ( optional )} - {isDone && Edit} + {isDone && Edit}
{/* Expanded form content for current step */} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_optional_params.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_optional_params.tsx index 2dcc0c69ed1..78077b0e768 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_optional_params.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_optional_params.tsx @@ -167,7 +167,7 @@ const DictField: React.FC = ({ field, fullFieldKey, control, val
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx index e5c26921043..df1d8d3436a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx @@ -20,7 +20,7 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt {label} - + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx index 7cdb5fc1d37..da264063839 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx @@ -201,7 +201,7 @@ const MCPPermissionManagement: React.FC = ({ Allow All LiteLLM Keys - +

@@ -218,7 +218,7 @@ const MCPPermissionManagement: React.FC = ({ Internal network only - +

@@ -236,7 +236,7 @@ const MCPPermissionManagement: React.FC = ({ Delegate auth to upstream (PKCE passthrough) - +

@@ -261,7 +261,7 @@ const MCPPermissionManagement: React.FC = ({ OAuth pass-through - +

@@ -296,7 +296,7 @@ const MCPPermissionManagement: React.FC = ({ MCP Access Groups - + } @@ -318,7 +318,7 @@ const MCPPermissionManagement: React.FC = ({ Extra Headers - + {mcpServer?.extra_headers && mcpServer.extra_headers.length > 0 && ( @@ -347,7 +347,7 @@ const MCPPermissionManagement: React.FC = ({ Static Headers - + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPSubmissionsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPSubmissionsTab.tsx index 6afa14adbe5..c6e47d4fb28 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPSubmissionsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPSubmissionsTab.tsx @@ -123,7 +123,7 @@ function ConfirmDialog({ action, serverName, isCurrentlyActive, onConfirm, onCan type="button" onClick={() => onConfirm(isApprove ? undefined : reviewNotes || undefined)} className={`flex-1 text-white text-sm font-medium py-2 rounded-md transition-colors ${ - isApprove ? "bg-success hover:bg-success" : "bg-destructive hover:bg-destructive" + isApprove ? "bg-success hover:bg-success/80" : "bg-destructive hover:bg-destructive/80" }`} > {isApprove ? "Approve" : "Reject"} @@ -235,7 +235,7 @@ function SubmissionRulesPanel({ requiredFields, onChange, onSave, isSaving }: Su await onSave(); setExpanded(false); }} - className="px-4 py-1.5 text-sm font-medium text-white bg-info hover:bg-info disabled:opacity-50 rounded-md transition-colors" + className="px-4 py-1.5 text-sm font-medium text-white bg-info hover:bg-info/80 disabled:opacity-50 rounded-md transition-colors" > {isSaving ? "Saving…" : "Save Rules"} @@ -322,7 +322,7 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe @@ -341,7 +341,7 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe @@ -383,7 +383,7 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe @@ -392,7 +392,7 @@ function MCPServerCard({ server, onApprove, onReject, requiredFields }: MCPServe diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx index c8407664e6e..c077a513757 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPToolsetsTab.tsx @@ -87,7 +87,9 @@ function MCPToolList({ serverId, serverName, accessToken, selectedTools, onToggl {serverName} {selectedSet.size > 0 && ( - {selectedSet.size} selected + + {selectedSet.size} selected + )} {expanded ? "▲" : "▼"} @@ -110,12 +112,14 @@ function MCPToolList({ serverId, serverName, accessToken, selectedTools, onToggl type="button" onClick={() => onToggle({ server_id: serverId, tool_name: tool.name })} className={`flex items-start justify-between px-3 py-2 rounded-lg text-left transition-colors ${ - selected ? "bg-purple-50 border border-purple-300" : "bg-card border border-border hover:bg-muted" + selected + ? "bg-purple-50 border border-purple-300 dark:bg-purple-950 dark:border-purple-700" + : "bg-card border border-border hover:bg-muted" }`} >

{tool.name}

@@ -125,7 +129,11 @@ function MCPToolList({ serverId, serverName, accessToken, selectedTools, onToggl

)}
- {selected && ✓} + {selected && ( + + ✓ + + )} ); })} @@ -271,15 +279,19 @@ function CreateToolsetModal({ open, onClose, onSave, accessToken, initialToolset key={idx} type="button" onClick={() => handleToggleTool(tool)} - className="w-full flex items-center justify-between px-3 py-1.5 rounded-lg border border-purple-200 bg-purple-50 hover:bg-destructive/10 hover:border-destructive/20 group transition-colors" + className="w-full flex items-center justify-between px-3 py-1.5 rounded-lg border border-purple-200 bg-purple-50 hover:bg-destructive/10 hover:border-destructive/20 group transition-colors dark:border-purple-800 dark:bg-purple-950" >
- + {displayToolName(serverPrefixById.get(tool.server_id), tool.tool_name)} - {tool.server_id.slice(0, 8)}… + + {tool.server_id.slice(0, 8)}… +
- ✕ + + ✕ + )) )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx index 3b960a3fcc0..76d667039d4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx @@ -54,7 +54,7 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt {label} - + ); @@ -190,7 +190,7 @@ const OAuthFormFields: React.FC = ({ href={docsUrl} target="_blank" rel="noopener noreferrer" - className="text-xs text-info hover:text-info ml-2 font-normal" + className="text-xs text-info hover:text-info/80 ml-2 font-normal" onClick={(e) => e.stopPropagation()} > Create OAuth App → diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx index 3340c180fd4..248776802cf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx @@ -70,7 +70,7 @@ const OpenAPIFormSection: React.FC = ({ OpenAPI Spec URL - + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx index 78a806b8553..a63821cd04c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx @@ -29,7 +29,7 @@ const OpenApiByokFields: React.FC = () => { BYOK (Bring Your Own Key) - + } @@ -65,7 +65,7 @@ const OpenApiByokFields: React.FC = () => { Access Description - + } @@ -85,7 +85,7 @@ const OpenApiByokFields: React.FC = () => { API Key Help URL - + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx index 223957ed446..854e67940a1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx @@ -38,7 +38,7 @@ const StdioConfiguration: React.FC = ({ isVisible, requ Stdio Configuration (JSON) - + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx index 6e2f81ef8f5..3dc2d62539d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx @@ -21,7 +21,7 @@ const TokenEndpointAuthMethodField: React.FC Token Endpoint Auth Method (optional) - + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx index ed36337cc6b..f081bae83f5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx @@ -26,7 +26,7 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt {label} - + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.tsx index dd4f623de91..d51fc5da5de 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.tsx @@ -275,7 +275,7 @@ export function ToolTestPanel({ className={`px-2 py-1 text-xs font-medium rounded transition-colors ${ viewMode === "formatted" ? "bg-success/15 text-success" - : "text-success hover:text-success" + : "text-success hover:text-success/80" }`} > Formatted @@ -283,7 +283,7 @@ export function ToolTestPanel({
))} -