From 250af8f027dbfaa9a7d2e1867cf53254056de60f Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:13:17 -0800 Subject: [PATCH 01/15] fix(spend)!: save streamed /v1/responses spend rows under the encrypted resp_ id the client received (#49493) * fix(responses): log the encrypted response id on streamed /v1/responses spend rows With response id security on, the stream hook encrypts the resp_ id on every frame, but logging had already received the SDK's own copy of the terminal response, so the spend row, the standard logging payload and callbacks got the managed id (or the chatcmpl id on a bridged stream) the client never saw. The hook now records the exact id it advertises on the request's logging object, and the logging layer stamps that id on the logged streamed response * fix(responses): return the id the client sent on GET and cancel, and keep read rows off the create row's id Every GET /v1/responses/{id} re-encrypted the response id, so a client that continued a bridged conversation with the id a GET returned referenced a ciphertext no spend row was filed under. The follow-up lost turn 1 and opened a new session. GET and cancel now answer with the exact id the client sent when it is an id this proxy issued and it decodes to the response the provider returned. A create with previous_response_id still gets a fresh id for its own response, and a raw provider id still gets a managed id Because a read now carries the create's id, a GET or cancel spend row is filed as _retrieve_ or _cancel_. That keeps one row per read, as before, and a read row can never take the create row's request_id, so the create row survives whichever row is written first and a follow-up replays turn 1 once. The background settlement row keeps _background_cost, and the cost poll's own status checks keep the response id * fix(responses): log a fresh id for each read and cancel, echo the held id only to its owner A GET or cancel of a stored response now logs a new encrypted id of its own, so callbacks and spend rows never reuse the create's id, while the owner of the id still gets back the exact id they sent. An admin reading someone else's response gets an id bound to the admin, as before. The spend row id suffix for reads and cancels is gone, so with response id security off the rows and session history match the default branch. A streamed GET follows the same rules. * fix(responses): echo the sent id to every reader that passes the response access check A GET, streamed GET or cancel of a response now answers with the id the caller sent whenever the caller passed the access check, not only when the caller is the user or team the id was issued to. An admin, the master key or a teammate who read someone else's streamed response got a fresh id back, and the session lookup could not reach the first turn from it, so a bridged follow-up through that id lost the conversation history Every read and cancel still logs a fresh id bound to the reader, never the create's id * fix(responses): keep a read or cancel row in session history only when it brings new output A GET, streamed GET or cancel sent with a session header joins the create's session, and its stored response repeats the create's output. The replay then sent the same assistant turn once per read. Session history now keeps an aget_responses or acancel_responses row only when its stored Responses output has an item id that no earlier kept row carried. A polled background response keeps the first poll that returns the finished output, since its queued create row has none, and later polls of the same output are dropped. Every other row keeps today's rule * fix(responses): replay each history item once, echo the sent id on delete, keep background cost files apart A create row now owns its output items in session history, and a read or cancel row adds only items no create and no earlier read carried, in one linear pass, so a background response polled while it grows, or a read that ends before its streamed create, replays the answer once and after its prompt DELETE /v1/responses/{id} answers with the id the caller sent, on a copy, and logs a fresh encrypted id like a read does Azure Data Lake Storage per-event files and unbatched GCS objects are named after the event's spend row id, so a streamed background create and its cost event, which now share the client's id, keep separate files The integration tests that decoded the managed id now match the exact id the client received * fix(responses): keep main's answer for a raw-id delete and main's storage names outside background cost events A DELETE by a raw provider id returns the provider's id as main did, and logging still records a fresh encrypted id. A streamed GET matches the held id by upstream response id, like a plain GET Only a background response's cost event gets the _background_cost suffix on its Azure Data Lake Storage file and unbatched GCS object, so batch retrieves and events with no id keep main's names An item a session read replays takes the newest read's copy at the first read's place, so an item a poll caught half written replays finished Tests pin the client's id over the router's continued response id, and the storage test no longer reads the clock after the upload * fix(spend): give background cost events their own OpenMeter and DynamoDB ids, replay id-less read items A streamed background create and its later cost event share the client's id, so OpenMeter dropped the second event and DynamoDB overwrote the create's item. Both now key the cost event as _background_cost, like its spend row and storage file. Session history again replays a read's output items that carry no id, as main did. * fix(integrations): keep OpenMeter and DynamoDB logging working without the proxy extras * fix(integrations): give background cost events their own ClickHouse request id and DeepEval span id A streamed background create and its later cost event now share the id the client received, so ClickHouse and DeepEval saw both events under one id. Route both through logged_event_key, the way the Azure storage, GCS, OpenMeter and DynamoDB loggers already do, so the cost event's ClickHouse request_id matches its spend row's `_background_cost` and its DeepEval span gets its own uuid. ClickHouse response_id stays the raw provider id. * test(deepeval): annotate the background cost span test's locals with Final * test(responses): run the DELETE wire test that pins the echoed client id DELETE /v1/responses/{id} now answers with the id the caller sent, so the skip that marked it as a known bug no longer applies. The test passes on the /v1, /openai/v1 and bare routes. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../azure_storage/azure_storage.py | 50 +- .../clickhouse/clickhouse_spend_logger.py | 7 +- litellm/integrations/deepeval/deepeval.py | 5 +- litellm/integrations/dynamodb.py | 18 +- litellm/integrations/gcs_bucket/gcs_bucket.py | 37 +- litellm/integrations/openmeter.py | 11 +- .../internal_call_metadata.py | 27 + litellm/litellm_core_utils/litellm_logging.py | 26 +- litellm/proxy/hooks/responses_id_security.py | 132 +++- .../spend_tracking/spend_tracking_utils.py | 33 +- .../session_handler.py | 98 ++- .../test_responses_retrieve_e2e.py | 3 - .../test_openai_sdk_client_paths.py | 15 +- .../test_langfuse_otel_metadata_chaos.py | 6 +- .../test_langfuse_otel_metadata_clients.py | 9 +- .../test_otel_conversation_id.py | 44 +- .../observability/test_signoz_delivery.py | 37 +- .../test_bedrock_gpt_responses_native_wire.py | 6 +- .../test_bedrock_mantle_output_limit_wire.py | 2 +- .../test_bedrock_model_discovery_wire.py | 9 +- ..._bedrock_runtime_chat_completions_chaos.py | 6 +- .../test_responses_lifecycle_wire.py | 4 - .../test_responses_stream_spend_row_id.py | 465 +++++++++++++ .../azure_storage/test_azure_storage.py | 181 ++++- .../test_clickhouse_spend_logger.py | 43 +- tests/unit/integrations/conftest.py | 36 +- .../gcs_bucket/test_gcs_bucket.py | 101 ++- tests/unit/integrations/test_deepeval.py | 35 + tests/unit/integrations/test_dynamodb.py | 52 ++ tests/unit/integrations/test_openmeter.py | 44 ++ .../test_litellm_logging.py | 34 + .../test_spend_tracking_utils.py | 17 + .../test_session_handler.py | 122 ++++ tests/unit/test_responses_id_security.py | 618 ++++++++++++++---- 34 files changed, 1981 insertions(+), 352 deletions(-) create mode 100644 tests/integration/spend/test_responses_stream_spend_row_id.py create mode 100644 tests/unit/integrations/test_dynamodb.py diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index 37b4d2d3937..2ff523571f2 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -1,7 +1,8 @@ import asyncio import os import time -from collections.abc import Callable +from collections.abc import Callable, Mapping +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from functools import cache from typing import Final @@ -15,6 +16,7 @@ from litellm.constants import ( AZURE_STORAGE_MSFT_VERSION, ) from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.litellm_core_utils.internal_call_metadata import logged_event_key from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id from litellm.llms.custom_httpx.http_handler import ( @@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str: return f"{name.translate(_ADLS_SAFE_NAME)}.json" +@dataclass(frozen=True, slots=True) +class AzureStorageLogQueueItem: + payload: StandardLoggingPayload + file_name: str + + +def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem: + payload_id: Final = payload.get("id") + return AzureStorageLogQueueItem( + payload=payload, + file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)), + ) + + @cache def _cached_credential_chain_token_provider() -> Callable[[], str]: return get_azure_ad_token_provider( @@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() - self.log_queue: list[StandardLoggingPayload] = [] + self.log_queue: list[AzureStorageLogQueueItem] = [] super().__init__(**kwargs, flush_lock=self.flush_lock) except Exception as e: verbose_logger.exception( @@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): def azure_storage_blob_endpoint(self) -> str: return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}" - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime): """ Async Log success events to Azure Blob Storage @@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger): if standard_logging_payload is None: raise ValueError("standard_logging_payload is not set") - self.log_queue.append(standard_logging_payload) + self.log_queue.append(_queued_log(kwargs, standard_logging_payload)) except Exception as e: verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime): """ Async Log failure events to Azure Blob Storage @@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): if standard_logging_payload is None: raise ValueError("standard_logging_payload is not set") - self.log_queue.append(standard_logging_payload) + self.log_queue.append(_queued_log(kwargs, standard_logging_payload)) except Exception as e: verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) @@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger): len(self.log_queue), ) - for payload in self.log_queue: - await self.async_upload_payload_to_azure_blob_storage(payload=payload) + for item in self.log_queue: + await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name) except Exception as e: verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e) - async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload): + async def async_upload_payload_to_azure_blob_storage( + self, payload: StandardLoggingPayload, file_name: str | None = None + ): """ Uploads the payload to Azure Blob Storage using a 3-step process: 1. Create file resource @@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger): """ try: if self.azure_storage_account_key: - await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload) + await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name) else: # Get a valid token instead of always requesting a new one await self.set_valid_azure_ad_token() async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) json_payload: Final = safe_dumps(payload) + "\n" # Add newline for each log entry payload_bytes: Final = json_payload.encode("utf-8") - filename: Final = adls_safe_file_name(payload.get("id")) + filename: Final = file_name or adls_safe_file_name(payload.get("id")) base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}" # Execute the 3-step upload process @@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger): self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS return self._service_client - async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload): + async def upload_to_azure_data_lake_with_azure_account_key( + self, payload: StandardLoggingPayload, file_name: str | None = None + ): """ Uploads the payload to Azure Data Lake using the Azure SDK @@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger): try: today: Final = datetime.now().strftime("%Y-%m-%d") - file_name: Final = adls_safe_file_name(payload.get("id")) + resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id")) await self._upload_bytes_with_account_key( - file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8") + file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8") ) - verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name) + verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name) except Exception as e: verbose_logger.exception("Error occurred: %s", e) diff --git a/litellm/integrations/clickhouse/clickhouse_spend_logger.py b/litellm/integrations/clickhouse/clickhouse_spend_logger.py index 1d01965cd2c..c8453456052 100644 --- a/litellm/integrations/clickhouse/clickhouse_spend_logger.py +++ b/litellm/integrations/clickhouse/clickhouse_spend_logger.py @@ -19,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger from litellm.integrations.clickhouse.context import is_lens_analysis from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE +from litellm.litellm_core_utils.internal_call_metadata import logged_event_key from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.tracing.types import SpendLogPayload, SpendLogRecord @@ -134,7 +135,7 @@ def spend_log_row_from_payload( usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({}) cache_read_tokens, cache_write_tokens = _cache_tokens(usage) trace_id, span_id = _find_traceparent(metadata, kwargs) - request_id = str(payload.get("id") or "") + response_id: Final = str(payload.get("id") or "") redact = litellm.turn_off_message_logging is True track_end_user: Final = not litellm.disable_end_user_cost_tracking completion_start_ms = _to_ms(payload.get("completionStartTime")) @@ -157,8 +158,8 @@ def spend_log_row_from_payload( ) ) return SpendLogRecord( - request_id=request_id, - response_id=strip_cache_hit_suffix(request_id), + request_id=logged_event_key(kwargs, response_id), + response_id=strip_cache_hit_suffix(response_id), provider_request_id=( get_provider_request_id(kwargs.get("response_headers")) or get_provider_request_id(hidden_params.get("additional_headers")) diff --git a/litellm/integrations/deepeval/deepeval.py b/litellm/integrations/deepeval/deepeval.py index 71d149e4014..e38b987d46f 100644 --- a/litellm/integrations/deepeval/deepeval.py +++ b/litellm/integrations/deepeval/deepeval.py @@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import ( to_zod_compatible_iso, validate_environment, ) +from litellm.litellm_core_utils.internal_call_metadata import logged_event_key # This file includes the custom callbacks for LiteLLM Proxy @@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger): ) else: output = str(standard_logging_object.get("error_string", "")) + logged_id: Final = standard_logging_object.get("id", uuid.uuid4()) + span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id return BaseApiSpan( - uuid=standard_logging_object.get("id", uuid.uuid4()), + uuid=span_id, name=("litellm_success_callback" if is_success else "litellm_failure_callback"), status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED), type=SpanApiType.LLM, diff --git a/litellm/integrations/dynamodb.py b/litellm/integrations/dynamodb.py index 6569e70329b..8672eb00ce5 100644 --- a/litellm/integrations/dynamodb.py +++ b/litellm/integrations/dynamodb.py @@ -9,6 +9,8 @@ from typing import Final, Protocol import litellm from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.internal_call_metadata import logged_event_key class _DynamoTable(Protocol): @@ -22,11 +24,13 @@ class _DynamoResource(Protocol): class DyanmoDBLogger: # Class variables or attributes - def __init__(self): + def __init__(self, dynamodb: _DynamoResource | None = None): # Instance variables import boto3 - self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"]) + self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource( + "dynamodb", region_name=os.environ["AWS_REGION_NAME"] + ) if litellm.dynamodb_table_name is None: raise ValueError( "LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=`" @@ -45,23 +49,23 @@ class DyanmoDBLogger: _async_log_event = async_log_event - def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): + def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose): try: print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}") # construct payload to send to DynamoDB # follows the same params as langfuse.py - litellm_params: Final = kwargs.get("litellm_params", {}) - metadata: Final = litellm_params.get("metadata", {}) or {} # if litellm_params['metadata'] == None + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {} messages: Final = kwargs.get("messages") optional_params: Final = kwargs.get("optional_params", {}) call_type: Final = kwargs.get("call_type", "litellm.completion") usage: Final = response_obj["usage"] - id: Final = response_obj.get("id", str(uuid.uuid4())) + response_id: Final = response_obj.get("id", str(uuid.uuid4())) # Build the initial payload payload: Final[dict[str, object]] = { - "id": id, + "id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id, "call_type": call_type, "startTime": start_time, "endTime": end_time, diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index 99cd8b31b63..7057e719764 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( sanitize_cloud_object_component, ) from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.internal_call_metadata import logged_event_key from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus from litellm.types.integrations.gcs_bucket import * from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload @@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): else: await self._send_individual_logs(items_to_process) - def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str: + def _get_object_name( + self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any + ) -> str: """ Get the object name to use for the current payload """ current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc)) + metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {} + gcs_log_id: Final = metadata.get("gcs_log_id") + safe_log_id: Final = sanitize_cloud_object_component( + gcs_log_id if isinstance(gcs_log_id, str) else None, fallback="" + ) + if safe_log_id: + return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}" if logging_payload.get("error_str", None) is not None: - object_name = self._generate_failure_object_name( - request_date_str=current_date, - ) - else: - object_name = self._generate_success_object_name( - request_date_str=current_date, - response_id=response_obj.get("id", ""), - ) - - # used for testing - _litellm_params: Final = kwargs.get("litellm_params", None) or {} - _metadata: Final = _litellm_params.get("metadata", None) or {} - if "gcs_log_id" in _metadata: - safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="") - if safe_log_id: - object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}" - - return object_name + return self._generate_failure_object_name(request_date_str=current_date) + raw_response_id: Final[object] = response_obj.get("id", "") + response_id: Final = raw_response_id if isinstance(raw_response_id, str) else "" + return self._generate_success_object_name( + request_date_str=current_date, + response_id=logged_event_key(kwargs, response_id), + ) async def get_request_response_payload( self, diff --git a/litellm/integrations/openmeter.py b/litellm/integrations/openmeter.py index db2fe386dec..918e849a086 100644 --- a/litellm/integrations/openmeter.py +++ b/litellm/integrations/openmeter.py @@ -3,12 +3,15 @@ import json import os +from collections.abc import Mapping from typing import Final import httpx import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.internal_call_metadata import logged_event_key from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_async_httpx_client, @@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger): if len(missing_keys) > 0: raise Exception(f"Missing keys={missing_keys} in environment.") - def _common_logic(self, kwargs: dict, response_obj): + def _common_logic(self, kwargs: Mapping[str, object], response_obj): call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id")) dt: Final = get_utc_datetime().isoformat() cost: Final = kwargs.get("response_cost", None) @@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger): # If no user provided directly, try to get it from token user_id if user_param is None: # Check if user_id is available from the API key metadata - litellm_params: Final = kwargs.get("litellm_params", {}) - metadata: Final = litellm_params.get("metadata", {}) + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {} user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None) if user_api_key_user_id is not None: @@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger): return { "specversion": "1.0", "type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"), - "id": call_id, + "id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id, "time": dt, "subject": subject, "source": "litellm-proxy", diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index 453d4358b35..138a01c6524 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -26,7 +26,12 @@ from litellm.constants import ( MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, NON_INFERENCE_CALL_TYPES, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN, @@ -35,6 +40,8 @@ from litellm.types.utils import ( BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"}) +BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost" + ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved" OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved" ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key" @@ -151,6 +158,26 @@ def is_unbilled_non_inference_call_from_params( return is_unbilled_non_inference_call(call_type, metadata, response) +def settles_background_response(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> bool: + return kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES or ( + call_type in NON_INFERENCE_CALL_TYPES + and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + ) + + +def logged_event_key(kwargs: Mapping[str, object], event_id: str) -> str: + call_type: Final = kwargs.get("call_type") + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + metadata: Final = ( + as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {} + ) + if not event_id or not settles_background_response( + call_type if isinstance(call_type, str) else "", metadata, kwargs + ): + return event_id + return f"{event_id}{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}" + + def sanitize_user_api_key_auth(auth: object) -> object: """Copy of the auth object with its budget reservation removed; the cost callback falls back to reading the reservation from inside the auth object.""" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5f909f639c4..37657a94f16 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -872,6 +872,7 @@ class Logging(LiteLLMLoggingBaseClass): # The model the proxy shows the client on streamed chunks. The logged streamed response carries it # once that response is priced, the same way a non-streamed response is logged self.client_facing_stream_model: str | None = None + self.client_facing_response_id: str | None = None self.zero_cost_warned: bool = False self.llm_caching_handler: LLMCachingHandler | None = None @@ -3238,6 +3239,23 @@ class Logging(LiteLLMLoggingBaseClass): return response return response.model_copy(update={"id": response_id}) + def _with_client_facing_response_id( + self, + response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, + ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: + response_id: Final = self.client_facing_response_id + if response_id is None or getattr(response, "id", None) == response_id: + return response + return response.model_copy(update={"id": response_id}) + + def _client_facing_streaming_response( + self, + response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, + ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: + return self._with_client_facing_response_id( + self._with_continued_response_id(self._with_client_facing_stream_model(response)) + ) + def _success_handler_helper_fn( self, result: object = None, @@ -3641,9 +3659,7 @@ class Logging(LiteLLMLoggingBaseClass): ) self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) - logged_streaming_response: Final = self._with_continued_response_id( - self._with_client_facing_stream_model(complete_streaming_response) - ) + logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) self.model_call_details["complete_streaming_response"] = logged_streaming_response self.capture_usage_before_logging(logged_streaming_response, start_time, end_time) ## STANDARDIZED LOGGING PAYLOAD @@ -4240,9 +4256,7 @@ class Logging(LiteLLMLoggingBaseClass): await self._prepare_baseline_cache_estimate(complete_streaming_response) - logged_streaming_response: Final = self._with_continued_response_id( - self._with_client_facing_stream_model(complete_streaming_response) - ) + logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) self.model_call_details["async_complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 771d68a0ea0..61ab1eb4771 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -7,22 +7,25 @@ instead of writing immediately on each request. from collections.abc import AsyncGenerator, Callable, Mapping from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, TypeVar from fastapi import HTTPException from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import LitellmUserRoles from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, ResponsesAPIResponse, ) +from litellm.types.responses.main import DeleteResponseResult from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums if TYPE_CHECKING: @@ -67,6 +70,8 @@ def _proxy_signing_key() -> str | None: _RESPONSE_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_ResponseObjectT = TypeVar("_ResponseObjectT", bound=BaseLiteLLMOpenAIResponseObject) +_AddressedResultT = TypeVar("_AddressedResultT", bound=ResponsesAPIResponse | DeleteResponseResult) def _response_payload(response_obj: object) -> Mapping[str, object] | None: @@ -77,9 +82,9 @@ def _response_payload(response_obj: object) -> Mapping[str, object] | None: def _rewrite_advertised_id( - event: BaseLiteLLMOpenAIResponseObject, + event: _ResponseObjectT, rewrite: Callable[[str], str], -) -> BaseLiteLLMOpenAIResponseObject: +) -> _ResponseObjectT: event_id: Final = getattr(event, "id", None) if isinstance(event_id, str) and event_id.startswith("resp_"): setattr(event, "id", rewrite(event_id)) @@ -116,6 +121,35 @@ def _streams_response_ids(request_route: str | None) -> bool: return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "") +_NO_HELD_IDS: Final[Mapping[str, str]] = MappingProxyType({}) + + +def _upstream_response_id(response_id: str) -> str: + return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response_id) + + +def _names_the_same_response(response_id: str | None, original_response_id: str) -> bool: + return response_id is not None and _upstream_response_id(response_id) == _upstream_response_id(original_response_id) + + +def _held_id_naming(response_id: str | None, held_ids: Mapping[str, str]) -> str | None: + return next( + ( + held_id + for original_response_id, held_id in held_ids.items() + if _names_the_same_response(response_id, original_response_id) + ), + None, + ) + + +def _id_sent_back(response: ResponsesAPIResponse | DeleteResponseResult, held_ids: Mapping[str, str]) -> str | None: + held_id: Final = _held_id_naming(response.id, held_ids) + if held_id is not None: + return held_id + return response.id if isinstance(response, DeleteResponseResult) else None + + class ResponsesIDSecurity(CustomLogger): def __init__( self, @@ -275,10 +309,18 @@ class ResponsesIDSecurity(CustomLogger): def _encrypt_response_id( self, - response: BaseLiteLLMOpenAIResponseObject, + response: _ResponseObjectT, user_api_key_dict: "UserAPIKeyAuth", request_cache: dict[str, str] | None = None, - ) -> BaseLiteLLMOpenAIResponseObject: + ) -> _ResponseObjectT: + encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache) + return response if encrypt is None else _rewrite_advertised_id(response, encrypt) + + def _response_id_encrypter( + self, + user_api_key_dict: "UserAPIKeyAuth", + request_cache: dict[str, str] | None, + ) -> Callable[[str], str] | None: signing_key: Final = self._get_signing_key() if signing_key is None: verbose_proxy_logger.debug( @@ -287,7 +329,7 @@ class ResponsesIDSecurity(CustomLogger): "Skipping response ID encryption. " "See: https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" ) - return response + return None def encrypt(original_id: str) -> str: cached: Final = request_cache.get(original_id) if request_cache is not None else None @@ -304,47 +346,73 @@ class ResponsesIDSecurity(CustomLogger): request_cache[original_id] = encrypted_id return encrypted_id - return _rewrite_advertised_id(response, encrypt) + return encrypt + + def _streamed_response_id_advertiser( + self, + user_api_key_dict: "UserAPIKeyAuth", + request_data: Mapping[str, object], + ) -> Callable[[str], str] | None: + if not _streams_response_ids(user_api_key_dict.request_route): + return None + if self._general_settings_reader().get("disable_responses_id_security", False): + return None + encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache={}) + if encrypt is None: + return None + logging_obj: Final = request_data.get("litellm_logging_obj") + held_ids: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD)) + + def advertise(original_id: str) -> str: + logged_id: Final = encrypt(original_id) + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.client_facing_response_id = logged_id + return _held_id_naming(original_id, held_ids) or logged_id + + return advertise + + def _ids_the_caller_holds(self, addressed_id: object) -> Mapping[str, str]: + if not isinstance(addressed_id, str) or not self._is_encrypted_response_id(addressed_id): + return _NO_HELD_IDS + original_response_id, _, _ = self._decrypt_response_id(addressed_id) + return MappingProxyType({original_response_id: addressed_id}) + + def _advertised_response( + self, + response: _AddressedResultT, + user_api_key_dict: "UserAPIKeyAuth", + request_data: Mapping[str, object], + ) -> _AddressedResultT: + sent_back_id: Final = _id_sent_back( + response, self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD)) + ) + logged_response: Final = self._encrypt_response_id(response, user_api_key_dict, request_cache=None) + return logged_response if sent_back_id is None else logged_response.model_copy(update={"id": sent_back_id}) async def async_post_call_success_hook( self, - data: dict, + data: dict[str, object], user_api_key_dict: "UserAPIKeyAuth", response: LLMResponseTypes, ) -> LLMResponseTypes: - """ - Queue response IDs for batch processing instead of writing directly to DB. - - This method adds response IDs to an in-memory queue, which are then - batch-processed by the DBSpendUpdateWriter during regular database update cycles. - """ general_settings: Final = self._general_settings_reader() if general_settings.get("disable_responses_id_security", False): return response - if isinstance(response, ResponsesAPIResponse): - response = cast( - ResponsesAPIResponse, - self._encrypt_response_id(response, user_api_key_dict, request_cache=None), - ) + if isinstance(response, ResponsesAPIResponse | DeleteResponseResult): + return self._advertised_response(response, user_api_key_dict, data) return response async def async_post_call_streaming_iterator_hook( - self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict + self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict[str, object] ) -> AsyncGenerator[BaseLiteLLMOpenAIResponseObject, None]: - general_settings: Final = self._general_settings_reader() - - # Create a request-scoped cache for consistent encryption across streaming chunks. - request_encryption_cache: Final[dict[str, str]] = {} - + advertise: Final = self._streamed_response_id_advertiser(user_api_key_dict, request_data) async for chunk in response: - if ( - isinstance(chunk, BaseLiteLLMOpenAIResponseObject) - and _streams_response_ids(user_api_key_dict.request_route) - and not general_settings.get("disable_responses_id_security", False) - ): - chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache) - yield chunk + yield ( + _rewrite_advertised_id(chunk, advertise) + if advertise is not None and isinstance(chunk, BaseLiteLLMOpenAIResponseObject) + else chunk + ) def decrypted_response_id(response_id: str) -> str: diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index ba5f873c4e9..8b0ff718fec 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -20,7 +20,6 @@ from litellm.constants import ( CALLER_SESSION_ID_METADATA_KEY, CLI_SESSION_KEY_PREFIX, EMPTY_MAPPING, - INTERNAL_CALL_ORIGIN_METADATA_KEY, KUBERNETES_POD_ROUTING_KEY, LITELLM_PROXY_MASTER_KEY_ALIAS, LITELLM_TRUNCATED_PAYLOAD_FIELD, @@ -30,7 +29,6 @@ from litellm.constants import ( MAX_SPEND_LOG_MODEL_NAME_LENGTH, MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB, MCP_SPEND_LOG_MODEL_PREFIX, - NON_INFERENCE_CALL_TYPES, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL, @@ -53,7 +51,11 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider from litellm.litellm_core_utils.hidden_params import served_from_cache -from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call +from litellm.litellm_core_utils.internal_call_metadata import ( + BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX, + is_unbilled_non_inference_call, + settles_background_response, +) from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, UsageObservation, @@ -79,13 +81,8 @@ from litellm.responses.litellm_completion_transformation.compaction_items import summary_only_compaction_content, with_compaction_summary, ) -from litellm.types.passthrough_endpoints.pass_through_endpoints import ( - SETTLES_BACKGROUND_RESPONSE_KWARG, - BackgroundResponseSettlement, -) from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag from litellm.types.utils import ( - BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, AzureSpillover, CallTypes, @@ -347,24 +344,21 @@ def _get_spend_logs_metadata( BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost" -BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost" -def _cost_row_request_id_suffix( - call_type: str, metadata: Mapping[str, object], settles_background_response: bool -) -> str: +def _cost_row_request_id_suffix(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> str: if call_type == CallTypes.aretrieve_batch.value: return BATCH_COST_REQUEST_ID_SUFFIX - if settles_background_response or ( - call_type in NON_INFERENCE_CALL_TYPES - and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN - ): + if settles_background_response(call_type, metadata, kwargs): return BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX return "" def get_spend_logs_id( - call_type: str, response_obj: dict, kwargs: Mapping[str, object], metadata: Mapping[str, object] = EMPTY_MAPPING + call_type: str, + response_obj: Mapping[str, object], + kwargs: Mapping[str, object], + metadata: Mapping[str, object] = EMPTY_MAPPING, ) -> str | None: standard_logging_payload = kwargs.get("standard_logging_object") candidate_ids: Final = ( @@ -377,10 +371,7 @@ def get_spend_logs_id( ) if resolved_id is None: return None - settles_background_response: Final = ( - kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES - ) - return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, settles_background_response)}" + return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, kwargs)}" _MISSING_ATTRIBUTE: Final = object() diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 7cf163e7ce8..7e6ee84e2b2 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -1,14 +1,16 @@ import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from itertools import chain +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.responses.litellm_completion_transformation.client_tool_search import ( bridge_client_tool_search, client_tool_search_loaded_tools, @@ -69,6 +71,90 @@ def _is_a_conversation_turn(spend_log: "SpendLogsPayload") -> bool: return spend_log.get("call_type") not in _MCP_TOOL_CALL_TYPES and not _is_a_failed_turn(spend_log) +_RESPONSE_READ_CALL_TYPES: Final = frozenset(("aget_responses", "acancel_responses")) + + +def _output_item_id(item: object) -> str | None: + match item: + case {"id": str() as item_id}: + return item_id + case _: + return None + + +def _is_a_response_read(spend_log: "SpendLogsPayload") -> bool: + return "call_type" in spend_log and spend_log["call_type"] in _RESPONSE_READ_CALL_TYPES + + +def _stored_response(spend_log: "SpendLogsPayload") -> object: + return spend_log.get("response") + + +def _output_items(spend_log: "SpendLogsPayload") -> Iterator[tuple[str | None, object]]: + output: Final = _stored_responses_api_output(_stored_response(spend_log)) or [] + return zip(map(_output_item_id, output), output) + + +def _identified_output_items(spend_log: "SpendLogsPayload") -> Iterator[tuple[str, object]]: + return ((item_id, item) for item_id, item in _output_items(spend_log) if item_id is not None) + + +def _output_item_ids(spend_log: "SpendLogsPayload") -> frozenset[str]: + return frozenset(item_id for item_id, _ in _identified_output_items(spend_log)) + + +def _first_index_holding_each_item(indexed_turns: Iterable[tuple[int, "SpendLogsPayload"]]) -> Mapping[str, int]: + return dict( + chain.from_iterable( + ((item_id, index) for item_id in _output_item_ids(turn)) for index, turn in reversed(tuple(indexed_turns)) + ) + ) + + +def _turn_replaying_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, int]: + indexed_turns: Final = tuple(enumerate(turns)) + first_read_holding: Final = _first_index_holding_each_item( + (index, turn) for index, turn in indexed_turns if _is_a_response_read(turn) + ) + first_other_turn_holding: Final = _first_index_holding_each_item( + (index, turn) for index, turn in indexed_turns if not _is_a_response_read(turn) + ) + return MappingProxyType({**first_read_holding, **first_other_turn_holding}) + + +def _latest_read_copy_of_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, object]: + return MappingProxyType( + dict(chain.from_iterable(_identified_output_items(turn) for turn in turns if _is_a_response_read(turn))) + ) + + +def _with_only_new_output( + index: int, turn: "SpendLogsPayload", turn_replaying: Mapping[str, int], latest_copy: Mapping[str, object] +) -> "SpendLogsPayload | None": + if not _is_a_response_read(turn): + return turn + stored_response: Final = as_str_mapping(_stored_response(turn)) + new_output: Final = [ + item if item_id is None else latest_copy[item_id] + for item_id, item in _output_items(turn) + if item_id is None or turn_replaying.get(item_id) == index + ] + if stored_response is None or not new_output: + return None + replayed_turn: Final[SpendLogsPayload] = {**turn, "response": {**stored_response, "output": new_output}} + return replayed_turn + + +def _conversation_turns(spend_logs: Sequence["SpendLogsPayload"]) -> tuple["SpendLogsPayload", ...]: + turns: Final = tuple(filter(_is_a_conversation_turn, spend_logs)) + turn_replaying: Final = _turn_replaying_each_item(turns) + latest_copy: Final = _latest_read_copy_of_each_item(turns) + replayed: Final = ( + _with_only_new_output(index, turn, turn_replaying, latest_copy) for index, turn in enumerate(turns) + ) + return tuple(turn for turn in replayed if turn is not None) + + _MCP_AUTO_EXECUTE_OUTPUT_TYPES: Final = frozenset(("mcp_tools_fetched", "tool_execution_results")) @@ -101,7 +187,7 @@ class StoredTurns: def _compaction_point( index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None ) -> _ReplayStart | None: - if response_holds_compaction_block(spend_log.get("response")): + if response_holds_compaction_block(_stored_response(spend_log)): return _ReplayStart(index=index, at_output=True) if proxy_server_request is not None and holds_compaction_summary(proxy_server_request.get("input")): return _ReplayStart(index=index, at_output=False) @@ -271,12 +357,10 @@ class ResponsesSessionHandler: previous_response_id: str, client_response_ids: tuple[str, ...] = (), ) -> StoredTurns: - spend_logs: Final = tuple( - spend_log - for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( + spend_logs: Final = _conversation_turns( + await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( previous_response_id, client_response_ids ) - if _is_a_conversation_turn(spend_log) ) return StoredTurns( spend_logs=spend_logs, @@ -378,7 +462,7 @@ class ResponsesSessionHandler: ############################################################ # Add Output messages for this Spend Log ############################################################ - _response_output: Final[object] = spend_log.get("response", "{}") + _response_output: Final = _stored_response(spend_log) if (responses_api_output := _stored_responses_api_output(_response_output)) is not None: chat_completion_message_history.extend( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( diff --git a/tests/e2e/llm_translation/test_responses_retrieve_e2e.py b/tests/e2e/llm_translation/test_responses_retrieve_e2e.py index 4a08259c028..0e954af0952 100644 --- a/tests/e2e/llm_translation/test_responses_retrieve_e2e.py +++ b/tests/e2e/llm_translation/test_responses_retrieve_e2e.py @@ -69,9 +69,6 @@ def _retrieve_response(proxy: ProxyClient, key: str, response_id: str) -> Respon class TestResponsesRetrieve: - @pytest.mark.skip( - reason="stage red: product gap (LIT-5446), retrieve returns a different id than the stored response (non-idempotent response-id re-encryption)" - ) @pytest.mark.covers("llm.responses.openai.basic.nonstream.works") @meta( Subject( diff --git a/tests/integration/compatibility/test_openai_sdk_client_paths.py b/tests/integration/compatibility/test_openai_sdk_client_paths.py index f8d7449038a..d6b6d1d460d 100644 --- a/tests/integration/compatibility/test_openai_sdk_client_paths.py +++ b/tests/integration/compatibility/test_openai_sdk_client_paths.py @@ -630,19 +630,6 @@ def _spend_row(identity: str) -> Mapping[str, JsonValue]: return rows[0] -def _spend_row_keyed_by_any(identities: frozenset[str]) -> Mapping[str, JsonValue]: - keys: Final = tuple(sorted(identities)) - placeholders: Final = ", ".join("%s" for _ in keys) - rows: Final = eventually( - lambda: read_rows( - f'SELECT request_id, call_type, model FROM "LiteLLM_SpendLogs" WHERE request_id IN ({placeholders})', keys - ), - lambda rows: len(rows) == 1, - seconds=70, - ) - return rows[0] - - def _spend_request_ids(identities: tuple[str, str, str]) -> frozenset[str]: rows: Final = eventually( lambda: read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id IN (%s, %s, %s)', identities), @@ -815,7 +802,7 @@ def test_r07_responses_stream_through_the_async_openai_sdk(gateway: Gateway) -> served: Final = _served(wire) assert _routes(served) == (("POST", "/v1/responses"),), served assert _body(served[0])["stream"] is True, served[0].body - assert _spend_row_keyed_by_any(response_identities(last.response.id))["model"] == DEPLOYMENT_MODEL + assert _spend_row(last.response.id)["model"] == DEPLOYMENT_MODEL def test_r08_completions_non_stream_through_the_sync_openai_sdk(gateway: Gateway) -> None: diff --git a/tests/integration/observability/test_langfuse_otel_metadata_chaos.py b/tests/integration/observability/test_langfuse_otel_metadata_chaos.py index bb386f6d41e..15bc076b8ba 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata_chaos.py +++ b/tests/integration/observability/test_langfuse_otel_metadata_chaos.py @@ -107,9 +107,11 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa ) payloads: Final = tuple(json.loads(line) for line in events) found: Final = next( - payload.get("response_id") or (payload.get("message") or {}).get("id") or payload.get("id") + (payload.get("response") or {}).get("id") + or (payload.get("message") or {}).get("id") + or payload.get("id") for payload in payloads - if payload.get("response_id") or payload.get("message") or payload.get("id") + if payload.get("response") or payload.get("message") or payload.get("id") ) return str(found) response: Final = rig.candidate.client.post(path, json=body, headers={"Authorization": f"Bearer {key}"}) diff --git a/tests/integration/observability/test_langfuse_otel_metadata_clients.py b/tests/integration/observability/test_langfuse_otel_metadata_clients.py index c2e0573b48a..fd2b9758449 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata_clients.py +++ b/tests/integration/observability/test_langfuse_otel_metadata_clients.py @@ -162,8 +162,13 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gatewa for line in stream.iter_lines() if line.startswith("data: ") and line != "data: [DONE]" ) - response_id: Final = next( - event["response_id"] for event in events if event.get("type") == "response.output_text.delta" + advertised: Final = frozenset( + event["response"]["id"] + for event in events + if event.get("type") in ("response.created", "response.completed") ) + assert len(advertised) == 1, events + response_id: Final = next(iter(advertised)) attrs: Final = exported_llm_span(rig.collector, response_id) + assert attrs["gen_ai.response.id"] == response_id, attrs assert_identity(attrs, chat_identity_fields(identity)) diff --git a/tests/integration/observability/test_otel_conversation_id.py b/tests/integration/observability/test_otel_conversation_id.py index d7f4de2961b..cd095991e53 100644 --- a/tests/integration/observability/test_otel_conversation_id.py +++ b/tests/integration/observability/test_otel_conversation_id.py @@ -1,5 +1,4 @@ import asyncio -import base64 import json import re import signal @@ -118,17 +117,6 @@ def _responses_reply(identity: str, stream: bool) -> Reply: ) -def _decoded_responses_id(identity: str) -> str: - try: - return base64.b64decode(identity.removeprefix("resp_").encode()).decode() - except (ValueError, UnicodeDecodeError): - return identity - - -def _canonical_id(identity: str) -> str: - return _decoded_responses_id(identity).rpartition("response_id:")[2] - - def _sse_events(text: str) -> tuple[dict[str, JsonValue], ...]: return tuple( json.loads(line[6:]) for line in text.splitlines() if line.startswith("data: ") and line != "data: [DONE]" @@ -170,8 +158,7 @@ class Collector: return tuple( attributes for attributes in self.attributes() - if isinstance(logged := attributes.get("gen_ai.response.id", {}).get("stringValue"), str) - and _canonical_id(logged) == _canonical_id(response_id) + if attributes.get("gen_ai.response.id", {}).get("stringValue") == response_id ) def conversation_ids(self, response_id: str) -> tuple[str | None, ...]: @@ -183,10 +170,6 @@ class Collector: def single_span(self, response_id: str) -> str | None: return eventually(lambda: self.conversation_ids(response_id), lambda values: len(values) == 1, seconds=30)[0] - def logged_id(self, response_id: str) -> str: - spans: Final = eventually(lambda: self.spans(response_id), lambda values: len(values) == 1, seconds=30) - return str(spans[0]["gen_ai.response.id"]["stringValue"]) - @pytest.fixture(scope="session") def collector() -> Iterator[Collector]: @@ -437,12 +420,13 @@ def test_responses_stream_raw_metadata_session_id_lands_as_conversation_id(rig: completed: Final = tuple(event for event in events if event["type"] == "response.completed") assert len(completed) == 1, events assert completed[0]["response"]["output"][0]["id"] == f"msg_resp_{marker}", completed - assert str(completed[0]["response"]["id"]).startswith("resp_"), completed + client_id: Final = str(completed[0]["response"]["id"]) + assert client_id.startswith("resp_"), completed assert rig.upstream_bodies(marker) == ( {"model": "gpt-4o-mini", "input": marker, "metadata": {"session_id": session}, "stream": True}, ) - assert rig.sink.single_span(f"resp_{marker}") == session - assert rig.spend_session(rig.sink.logged_id(f"resp_{marker}")) == session + assert rig.sink.single_span(client_id) == session + assert rig.spend_session(client_id) == session @pytest.mark.covers("other.observability.otel.conversation_id_from_metadata_chat_raw") @@ -689,15 +673,11 @@ def _burst(rig: Rig, count: int, session_for: Mapping[int, str]) -> tuple[tuple[ return tuple(pool.map(one, range(count))) -def _is_encrypted_responses_id(identity: str) -> bool: - return identity.startswith("resp_") and _decoded_responses_id(identity) == identity - - def _landed(rig: Rig, expected: Mapping[str, str]) -> dict[str, tuple[str, ...]]: spans: Final = rig.sink.attributes() return { session: tuple( - _canonical_id(str(attributes["gen_ai.response.id"]["stringValue"])) + str(attributes["gen_ai.response.id"]["stringValue"]) for attributes in spans if attributes.get(CONVERSATION, {}).get("stringValue") == session and "gen_ai.response.id" in attributes ) @@ -707,21 +687,13 @@ def _landed(rig: Rig, expected: Mapping[str, str]) -> dict[str, tuple[str, ...]] def _assert_exactly_once(rig: Rig, expected: Mapping[str, str], landed: Mapping[str, tuple[str, ...]]) -> None: spend: Final = eventually( - lambda: { - session: tuple(_canonical_id(identity) for identity in rig.spend_request_ids(session)) - for session in expected.values() - }, + lambda: {session: rig.spend_request_ids(session) for session in expected.values()}, lambda rows: all(len(values) >= 1 for values in rows.values()), seconds=70, ) assert landed == spend, (landed, spend) assert all(len(values) == 1 for values in landed.values()), landed - caller_visible: Final = { - session: (_canonical_id(identity),) - for identity, session in expected.items() - if not _is_encrypted_responses_id(identity) - } - assert {session: landed[session] for session in caller_visible} == caller_visible, landed + assert landed == {session: (identity,) for identity, session in expected.items()}, landed @pytest.mark.covers("other.observability.otel.conversation_id_sink_outage_recovers_exactly_once") diff --git a/tests/integration/observability/test_signoz_delivery.py b/tests/integration/observability/test_signoz_delivery.py index e25126e4fa8..93539d7641b 100644 --- a/tests/integration/observability/test_signoz_delivery.py +++ b/tests/integration/observability/test_signoz_delivery.py @@ -1,5 +1,4 @@ import asyncio -import base64 import json import re import signal @@ -9,7 +8,6 @@ from collections import deque from collections.abc import Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass -from functools import partial from pathlib import Path from typing import Final @@ -137,17 +135,6 @@ def _upstream(request: Request) -> Reply: return _chat_reply(f"chatcmpl-{marker}", stream) -def _decoded_responses_id(identity: str) -> str: - try: - return base64.b64decode(identity.removeprefix("resp_").encode()).decode() - except (ValueError, UnicodeDecodeError): - return identity - - -def _canonical_id(identity: str) -> str: - return _decoded_responses_id(identity).rpartition("response_id:")[2] - - def _sse_events(text: str) -> tuple[dict[str, JsonValue], ...]: return tuple( object_value(JSON.validate_json(line[6:])) @@ -232,8 +219,7 @@ class Collector: return tuple( span for span in self.spans() - if RESPONSE_ID in span.attributes - and _canonical_id(span.attributes[RESPONSE_ID]) == _canonical_id(response_id) + if RESPONSE_ID in span.attributes and span.attributes[RESPONSE_ID] == response_id ) def single_span(self, response_id: str, *, elsewhere: "Collector | None" = None) -> Span: @@ -249,11 +235,8 @@ class Collector: def landed(self, response_ids: Sequence[str]) -> dict[str, int]: spans: Final = self.spans() return { - _canonical_id(identity): sum( - 1 - for span in spans - if RESPONSE_ID in span.attributes - and _canonical_id(span.attributes[RESPONSE_ID]) == _canonical_id(identity) + identity: sum( + 1 for span in spans if RESPONSE_ID in span.attributes and span.attributes[RESPONSE_ID] == identity ) for identity in response_ids } @@ -448,7 +431,7 @@ def _assert_operator_span(rig: Rig, response_id: str, marker: str) -> Span: span: Final = rig.sink.single_span(response_id) assert span.target == "/v1/traces", span assert span.ingestion_key == OPERATOR_KEY, span - assert rig.tenant_sink.landed((response_id,)) == {_canonical_id(response_id): 0} + assert rig.tenant_sink.landed((response_id,)) == {response_id: 0} bodies: Final = rig.upstream_bodies(marker) assert len(bodies) == 1, bodies assert "signoz" not in json.dumps(bodies[0]).replace(marker, ""), bodies[0] @@ -522,7 +505,7 @@ def test_responses_stream_raw_httpx_span_lands_once_after_the_stream_is_consumed marker: Final = _marker() response: Final = rig.proxy.request("POST", "/v1/responses", {"model": rig.model, "input": marker, "stream": True}) assert response.status_code == 200, response.text - _assert_operator_span(rig, _responses_id(response, marker), marker) + _assert_operator_span(rig, _responses_id(response), marker) def test_v2_flag_on_still_delivers_the_operator_span_with_the_ingestion_key(v2_rig: Rig) -> None: @@ -821,7 +804,7 @@ def _chat_id(response: httpx.Response) -> str: return next(iter(identities)) -def _responses_id(response: httpx.Response, marker: str) -> str: +def _responses_id(response: httpx.Response) -> str: if not _is_event_stream(response): return _body_id(response) completed: Final = tuple( @@ -830,7 +813,7 @@ def _responses_id(response: httpx.Response, marker: str) -> str: if event.get("type") == "response.completed" ) assert len(completed) == 1 and completed[0].startswith("resp_"), response.text - return f"resp_{marker}" + return completed[0] def _message_id(response: httpx.Response) -> str: @@ -852,7 +835,7 @@ def _burst(rig: Rig, count: int) -> tuple[tuple[int, str, str | None], ...]: stream: Final = index % 2 == 0 path, body, identity_of = ( ("/v1/chat/completions", {"model": rig.model, "messages": [{"role": "user", "content": marker}]}, _chat_id), - ("/v1/responses", {"model": rig.model, "input": marker}, partial(_responses_id, marker=marker)), + ("/v1/responses", {"model": rig.model, "input": marker}, _responses_id), ( "/v1/messages", {"model": rig.model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}]}, @@ -874,8 +857,8 @@ def _assert_exactly_once(rig: Rig, identities: Sequence[str]) -> None: landed: Final = eventually( lambda: rig.sink.landed(identities), lambda seen: all(count >= 1 for count in seen.values()), seconds=80 ) - assert landed == {_canonical_id(identity): 1 for identity in identities}, landed - charged: Final = frozenset(identity for identity in identities if not identity.startswith("resp_")) + assert landed == {identity: 1 for identity in identities}, landed + charged: Final = frozenset(identities) spend: Final = eventually( lambda: read_rows( 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s::text[])', diff --git a/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py b/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py index 3d6eb7fcaff..0788b9ac85a 100644 --- a/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py +++ b/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py @@ -62,12 +62,12 @@ def _body(request: Request) -> dict[str, JsonValue]: # ciphertext the caller received, because the spend row id is read from response_obj["id"] before the # ResponsesIDSecurity hook rewrites it in place; the row is looked up under both ids until that ordering is fixed on # main -def _spend_row(client_id: str, issued_id: str) -> dict[str, JsonValue]: +def _spend_row(*request_ids: str) -> dict[str, JsonValue]: rows: Final = eventually( lambda: read_rows( 'SELECT model_group, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" ' "WHERE request_id = ANY(%s)", - ([client_id, issued_id],), # pyright: ignore[reportArgumentType] # psycopg adapts the list to a text array + (list(request_ids),), # pyright: ignore[reportArgumentType] # psycopg adapts the list to a text array ), lambda found: len(found) == 1, seconds=70, @@ -120,4 +120,4 @@ async def test_async_openai_sdk_responses_stream_is_served_by_the_native_respons assert issued.upstream == f"resp_upstream_{marker}", completed.response.id request: Final = _native_request(wire) assert _body(request) == {"model": GPT, "input": _prompt(marker), "stream": True}, request.body - assert _spend_row(completed.response.id, issued.issued) == _success_row(model) + assert _spend_row(completed.response.id) == _success_row(model) diff --git a/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py b/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py index 89ad226761e..5aede070896 100644 --- a/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py @@ -516,7 +516,7 @@ def test_responses_max_output_tokens_is_forwarded_natively_on_both_legs(gateway: for attempt in attempts: assert attempt.body["max_output_tokens"] == 1000 and _limits(attempt.body) == (None, None), attempt.body _assert_logged(issued, model, "success") - _assert_logged(_managed_response_id(streamed_issued), model, "success") + _assert_logged(streamed_issued, model, "success") def test_region_prefixed_deployment_on_a_v1_base_sends_the_bare_model_with_max_completion_tokens( diff --git a/tests/integration/providers/test_bedrock_model_discovery_wire.py b/tests/integration/providers/test_bedrock_model_discovery_wire.py index 3bf4583e736..2871e51de5e 100644 --- a/tests/integration/providers/test_bedrock_model_discovery_wire.py +++ b/tests/integration/providers/test_bedrock_model_discovery_wire.py @@ -301,11 +301,7 @@ def _spend_row_id(rows: Sequence[Mapping[str, JsonValue]], tag: str) -> str: def _landed_once(served: tuple[tuple[str, str, bool], ...], tags: tuple[str, ...]) -> None: - """One spend row per tagged call, carrying the id the client saw. - - FIXME: a streamed /v1/responses row carries the managed `resp_` id LiteLLM built before the proxy encrypted the - advertised one, so that row is matched by tag alone until the spend log reads the id the client received. - """ + """One spend row per tagged call, carrying the id the client saw.""" rows: Final = eventually( lambda: read_rows( 'SELECT request_id, request_tags FROM "LiteLLM_SpendLogs" WHERE request_tags ?| %s', @@ -316,9 +312,6 @@ def _landed_once(served: tuple[tuple[str, str, bool], ...], tags: tuple[str, ... ) for (identity, path, stream), tag in zip(served, tags, strict=True): row_id: Final = _spend_row_id(rows, tag) - if path == "/v1/responses" and stream: - assert row_id.startswith("resp_"), (row_id, identity) - continue assert row_id == identity, (path, stream, row_id, identity) diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py index 3cadc1b5ccf..42950fb18c3 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py @@ -172,7 +172,11 @@ def _upstream_id_inside(row_id: str) -> str | None: def _row_belongs_to(row_id: str, served: _Served) -> bool: if row_id == _response_id(served): return True - return served.call.endpoint == "responses" and _upstream_id_inside(row_id) == f"resp_upstream_{served.call.marker}" + return ( + served.call.endpoint == "responses" + and not served.call.stream + and _upstream_id_inside(row_id) == f"resp_upstream_{served.call.marker}" + ) def _assert_each_success_landed_once(rows: list[dict[str, JsonValue]], served: tuple[_Served, ...]) -> None: diff --git a/tests/integration/providers/test_responses_lifecycle_wire.py b/tests/integration/providers/test_responses_lifecycle_wire.py index 02269885b44..ac5b4dc6d5c 100644 --- a/tests/integration/providers/test_responses_lifecycle_wire.py +++ b/tests/integration/providers/test_responses_lifecycle_wire.py @@ -203,9 +203,6 @@ def test_retrieve_routes_to_the_deployment_that_created_the_response( @pytest.mark.parametrize("prefix", _ALIASES) def test_retrieve_returns_the_id_the_client_holds(gateway: Gateway, prefix: str) -> None: - pytest.skip( - "BUG: GET /v1/responses/{id} returns a re-encrypted id that differs byte-for-byte from the id the client sent" - ) raw_id: Final = f"resp_retrieve_id_{uuid.uuid4().hex}" def respond(request: Request) -> Reply: @@ -697,7 +694,6 @@ def test_delete_returns_the_client_held_id_not_the_provider_id( gateway: Gateway, prefix: str, ) -> None: - pytest.skip("BUG: DELETE /v1/responses/{id} returns the raw provider id instead of the proxy-issued id") raw_id: Final = f"resp_delete_id_{uuid.uuid4().hex}" def respond(request: Request) -> Reply: diff --git a/tests/integration/spend/test_responses_stream_spend_row_id.py b/tests/integration/spend/test_responses_stream_spend_row_id.py new file mode 100644 index 00000000000..58ed9f7785f --- /dev/null +++ b/tests/integration/spend/test_responses_stream_spend_row_id.py @@ -0,0 +1,465 @@ +import json +from collections.abc import Mapping +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from pathlib import Path +from types import MappingProxyType +from typing import Final +from uuid import uuid4 + +import httpx +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, list_value, object_value, string_value +from integration._support.process import owned_proxy +from integration._support.responses_stream import frame, response_object +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_NATIVE_MODEL: Final = "openai/gpt-5.6" +_BRIDGED_MODEL: Final = "deepseek/deepseek-v4-pro" +_ANSWER: Final = "streamed answer" +_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35} +_JSON_LIST: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _sse(payload: dict[str, JsonValue]) -> bytes: + return f"data: {json.dumps(payload)}\n\n".encode() + + +def _native_completed(upstream_id: str) -> dict[str, JsonValue]: + message: Final[dict[str, JsonValue]] = { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": _ANSWER, "annotations": []}], + } + return response_object(upstream_id, "completed", output=[message], usage=_USAGE) + + +def _native_stream(upstream_id: str) -> Reply: + return Reply( + content_type="text/event-stream", + chunks=( + frame( + { + "type": "response.created", + "sequence_number": 0, + "response": response_object(upstream_id, "in_progress"), + } + ), + frame( + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": _ANSWER, + } + ), + frame({"type": "response.completed", "sequence_number": 2, "response": _native_completed(upstream_id)}), + ), + ) + + +def _chat_stream(upstream_id: str, model: str) -> Reply: + def chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> dict[str, JsonValue]: + return { + "id": upstream_id, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + return Reply( + content_type="text/event-stream", + chunks=( + _sse(chunk({"role": "assistant", "content": _ANSWER}, None)), + _sse( + { + **chunk({}, "stop"), + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + } + ), + b"data: [DONE]\n\n", + ), + ) + + +def _respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + if request.method == "GET": + stored_id: Final = request.target.removeprefix("/v1/responses/") + assert stored_id.startswith("resp_upstream-"), request.target + return Reply(body=json.dumps(_native_completed(stored_id)).encode()) + if request.target.endswith("/cancel"): + cancelled_id: Final = request.target.removeprefix("/v1/responses/").removesuffix("/cancel") + assert cancelled_id.startswith("resp_upstream-"), request.target + return Reply(body=json.dumps(response_object(cancelled_id, "cancelled")).encode()) + body: Final = JSON_OBJECT.validate_json(request.body) + upstream_id: Final = f"upstream-{uuid4().hex}" + if request.target == "/v1/responses" and body.get("stream") is not True: + return Reply(body=json.dumps(_native_completed(f"resp_{upstream_id}")).encode()) + assert body.get("stream") is True, body + if request.target == "/v1/responses": + return _native_stream(f"resp_{upstream_id}") + assert request.target == "/v1/chat/completions", request.target + return _chat_stream(f"chatcmpl-{upstream_id}", string_value(body["model"])) + + +def _terminal_response_id( + gateway: Gateway, key: str, body: dict[str, JsonValue], headers: Mapping[str, str] | None = None +) -> str: + response: Final = gateway.request("POST", "/v1/responses", body, key=key, headers=headers) + assert response.status_code == 200, response.text + frames: Final = tuple( + JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + assert frames and frames[-1]["type"] == "response.completed", response.text + advertised: Final = tuple( + dict.fromkeys(string_value(object_value(event["response"])["id"]) for event in frames if "response" in event) + ) + assert len(advertised) == 1, advertised + return advertised[0] + + +def _created_response_id( + gateway: Gateway, key: str, body: dict[str, JsonValue], headers: Mapping[str, str] | None = None +) -> str: + if body.get("stream") is True: + return _terminal_response_id(gateway, key, body, headers) + response: Final = gateway.request("POST", "/v1/responses", body, key=key, headers=headers) + assert response.status_code == 200, response.text + return string_value(JSON_OBJECT.validate_json(response.content)["id"]) + + +def _read_and_cancel( + gateway: Gateway, key: str, response_id: str, headers: Mapping[str, str] | None = None +) -> tuple[httpx.Response, ...]: + responses: Final = tuple( + gateway.request(method, path, key=key, headers=headers) + for method, path in ( + ("GET", f"/v1/responses/{response_id}"), + ("GET", f"/v1/responses/{response_id}"), + ("POST", f"/v1/responses/{response_id}/cancel"), + ) + ) + assert [response.status_code for response in responses] == [200, 200, 200], [r.text for r in responses] + return responses + + +def _spend_logs_lookup(gateway: Gateway, request_id: str) -> list[JsonValue]: + response: Final = gateway.request("GET", "/spend/logs", params={"request_id": request_id}) + assert response.status_code == 200, response.text + return [row["request_id"] for row in _JSON_LIST.validate_json(response.content)] + + +def _request_ids_for_key(gateway: Gateway, key: str, count: int) -> list[str]: + rows: Final = eventually( + lambda: gateway.rows( + 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key=%s ORDER BY "endTime"', + (sha256(key.encode()).hexdigest(),), + ), + lambda found: len(found) >= count, + seconds=70, + ) + return [string_value(row["request_id"]) for row in rows] + + +def _logs_page_search(gateway: Gateway, client_id: str) -> list[str]: + now: Final = datetime.now(timezone.utc) + page: Final = gateway.get( + "/spend/logs/ui", + params={ + "search": client_id, + "start_date": (now - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + "end_date": (now + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + }, + ) + return [string_value(object_value(row)["request_id"]) for row in list_value(page["data"])] + + +@pytest.mark.timeout(120) +@pytest.mark.parametrize("litellm_model", [_NATIVE_MODEL, _BRIDGED_MODEL], ids=["native", "bridge"]) +def test_streamed_responses_spend_row_is_filed_under_the_id_the_client_saw( + gateway: Gateway, litellm_model: str +) -> None: + with wire_server(_respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=litellm_model, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[model]) + + client_id: Final = _terminal_response_id( + gateway, key, {"model": model, "input": f"marker-{uuid4().hex}", "stream": True} + ) + + assert client_id.startswith("resp_"), client_id + assert _request_ids_for_key(gateway, key, 1) == [client_id] + assert _logs_page_search(gateway, client_id) == [client_id] + assert _spend_logs_lookup(gateway, client_id) == [client_id] + + +def _proxy_config( + directory: Path, + general_settings: Mapping[str, JsonValue], + litellm_settings: Mapping[str, JsonValue] = MappingProxyType({}), +) -> Path: + stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + path: Final = directory / "responses-spend.yaml" + path.write_text( + yaml.safe_dump( + { + **stock, + "general_settings": {**object_value(stock["general_settings"]), **general_settings}, + "litellm_settings": {**object_value(stock["litellm_settings"]), **litellm_settings}, + } + ) + ) + return path + + +def _prompt_storing_config(directory: Path) -> Path: + return _proxy_config(directory, {"store_prompts_in_spend_logs": True}) + + +def test_a_follow_up_turn_on_a_streamed_bridged_response_replays_the_first_turn( + gateway: Gateway, tmp_path: Path +) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing, + storing.scenario() as scenario, + ): + model: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[model]) + first_id: Final = _terminal_response_id(storing, key, {"model": model, "input": first_prompt, "stream": True}) + assert _request_ids_for_key(storing, key, 1) == [first_id] + wire.drain() + + second_id: Final = _terminal_response_id( + storing, + key, + {"model": model, "input": "second turn", "previous_response_id": first_id, "stream": True}, + ) + + replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"]) + assert [(message["role"], message["content"]) for message in replayed] == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ], replayed + assert _request_ids_for_key(storing, key, 2) == [first_id, second_id] + + +@pytest.mark.timeout(180) +@pytest.mark.parametrize("stream_first_turn", [True, False], ids=["streamed-first-turn", "non-streamed-first-turn"]) +def test_a_follow_up_through_the_id_a_retrieve_returned_replays_the_first_turn_once( + gateway: Gateway, tmp_path: Path, stream_first_turn: bool +) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing, + storing.scenario() as scenario, + ): + native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[native, bridged]) + first_id: Final = _created_response_id( + storing, key, {"model": native, "input": first_prompt, "stream": stream_first_turn} + ) + + retrieved: Final = storing.request("GET", f"/v1/responses/{first_id}", key=key) + + assert retrieved.status_code == 200, retrieved.text + retrieved_id: Final = string_value(JSON_OBJECT.validate_json(retrieved.content)["id"]) + assert retrieved_id == first_id + stored: Final = _request_ids_for_key(storing, key, 2) + assert stored.count(first_id) == 1, stored + assert len(set(stored)) == 2, stored + assert _spend_logs_lookup(storing, first_id) == [first_id] + wire.drain() + + _terminal_response_id( + storing, + key, + {"model": bridged, "input": "second turn", "previous_response_id": retrieved_id, "stream": True}, + ) + + replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"]) + assert [(message["role"], message["content"]) for message in replayed] == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ], replayed + + +def _replayed_turns(wire_body: bytes) -> list[tuple[JsonValue, JsonValue]]: + replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire_body)["messages"]) + return [(message["role"], message["content"]) for message in replayed] + + +@pytest.mark.timeout(180) +@pytest.mark.parametrize("reader", ["proxy-admin", "master-key"]) +def test_a_reader_other_than_the_creator_continues_through_the_id_its_retrieve_returned( + gateway: Gateway, tmp_path: Path, reader: str +) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing, + storing.scenario() as scenario, + ): + native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + creator: Final = scenario.key(user_id=scenario.user(), models=[native, bridged]) + reader_key: Final = ( + scenario.key(user_id=scenario.user(user_role="proxy_admin"), models=[native, bridged]) + if reader == "proxy-admin" + else storing.key + ) + first_id: Final = _terminal_response_id( + storing, creator, {"model": native, "input": first_prompt, "stream": True} + ) + assert _request_ids_for_key(storing, creator, 1) == [first_id] + + retrieved: Final = storing.request("GET", f"/v1/responses/{first_id}", key=reader_key) + + assert retrieved.status_code == 200, retrieved.text + retrieved_id: Final = string_value(JSON_OBJECT.validate_json(retrieved.content)["id"]) + wire.drain() + _terminal_response_id( + storing, + reader_key, + {"model": bridged, "input": "second turn", "previous_response_id": retrieved_id, "stream": True}, + ) + assert _replayed_turns(wire.drain()[-1].body) == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ] + assert retrieved_id == first_id + + +@pytest.mark.timeout(180) +def test_reads_and_cancels_sent_with_a_session_header_add_no_replayed_turns(gateway: Gateway, tmp_path: Path) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + session: Final = {"x-litellm-session-id": f"session-{uuid4().hex}"} + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing, + storing.scenario() as scenario, + ): + native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[native, bridged]) + first_id: Final = _created_response_id(storing, key, {"model": native, "input": first_prompt}, session) + + _read_and_cancel(storing, key, first_id, session) + + assert len(_request_ids_for_key(storing, key, 4)) == 4 + wire.drain() + _terminal_response_id( + storing, + key, + {"model": bridged, "input": "second turn", "previous_response_id": first_id, "stream": True}, + session, + ) + assert _replayed_turns(wire.drain()[-1].body) == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ] + + +@pytest.mark.timeout(180) +def test_each_read_and_cancel_reaches_callbacks_and_spend_logs_under_an_id_of_its_own( + gateway: Gateway, tmp_path: Path +) -> None: + logged_call_types: Final = frozenset({"aresponses", "aget_responses", "acancel_responses"}) + with ( + wire_server(_respond) as wire, + wire_server(lambda _: Reply()) as sink, + owned_proxy( + gateway, + tmp_path, + {"GENERIC_LOGGER_ENDPOINT": sink.url}, + config=_proxy_config(tmp_path, {}, {"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1}), + ) as logging_proxy, + logging_proxy.scenario() as scenario, + ): + model: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[model]) + first_id: Final = _created_response_id(logging_proxy, key, {"model": model, "input": f"marker-{uuid4().hex}"}) + + reads: Final = _read_and_cancel(logging_proxy, key, first_id) + + assert [string_value(JSON_OBJECT.validate_json(read.content)["id"]) for read in reads] == [first_id] * 3 + batches: Final[list[bytes]] = [] # mutable-ok: drain() consumes, later polls keep earlier batches + + def delivered() -> tuple[tuple[str, str], ...]: + batches.extend(batch.body for batch in sink.drain()) + return tuple( + (string_value(event["call_type"]), string_value(event["id"])) + for event in (object_value(item) for batch in batches for item in _JSON_LIST.validate_json(batch)) + if event["call_type"] in logged_call_types + ) + + events: Final = eventually(delivered, lambda found: len(found) >= 4, seconds=30) + read_ids: Final = [logged_id for call_type, logged_id in events if call_type != "aresponses"] + assert [logged_id for call_type, logged_id in events if call_type == "aresponses"] == [first_id], events + assert len(read_ids) == 3, events + assert first_id not in read_ids, events + assert len(set(read_ids)) == 3, events + assert sorted(_request_ids_for_key(logging_proxy, key, 4)) == sorted([first_id, *read_ids]) + + +@pytest.mark.timeout(240) +def test_with_id_security_off_reads_and_cancels_add_no_rows_and_no_replayed_turns( + gateway: Gateway, tmp_path: Path +) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + session: Final = {"x-litellm-session-id": f"session-{uuid4().hex}"} + with ( + wire_server(_respond) as wire, + owned_proxy( + gateway, + tmp_path, + {}, + config=_proxy_config( + tmp_path, {"store_prompts_in_spend_logs": True, "disable_responses_id_security": True} + ), + ) as insecure, + insecure.scenario() as scenario, + ): + native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[native, bridged]) + first_id: Final = _created_response_id(insecure, key, {"model": native, "input": first_prompt}, session) + + reads: Final = _read_and_cancel(insecure, key, first_id, session) + + assert [string_value(JSON_OBJECT.validate_json(read.content)["id"]) for read in reads] == [first_id] * 3 + later_id: Final = _created_response_id(insecure, key, {"model": native, "input": "a later request"}) + assert _request_ids_for_key(insecure, key, 2) == [first_id, later_id] + wire.drain() + + _terminal_response_id( + insecure, + key, + {"model": bridged, "input": "second turn", "previous_response_id": first_id, "stream": True}, + session, + ) + + replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"]) + assert [(message["role"], message["content"]) for message in replayed] == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ], replayed diff --git a/tests/unit/integrations/azure_storage/test_azure_storage.py b/tests/unit/integrations/azure_storage/test_azure_storage.py index 0f0ce24ff84..ee12291e7a0 100644 --- a/tests/unit/integrations/azure_storage/test_azure_storage.py +++ b/tests/unit/integrations/azure_storage/test_azure_storage.py @@ -5,19 +5,28 @@ import re import sys import threading from datetime import datetime, timedelta, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest -from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS +from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS, INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.azure_storage import azure_storage as azure_storage_module from litellm.integrations.azure_storage.azure_storage import ( AzureBlobStorageLogger, _cached_credential_chain_token_provider, adls_safe_file_name, ) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.secret_managers.get_azure_ad_token_provider import AzureCredentialType -from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload +from litellm.types.utils import ( + BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, + StandardAuditLogPayload, + StandardLoggingPayload, +) @pytest.fixture @@ -648,3 +657,171 @@ async def test_audit_log_is_uploaded_to_a_dated_audit_logs_path_with_an_account_ assert json.loads(uploaded) == audit_log assert file_client.append_data.await_args.kwargs == {"data": uploaded, "offset": 0, "length": len(uploaded)} file_client.flush_data.assert_awaited_once_with(position=len(uploaded), offset=0) + + +BACKGROUND_RESPONSE_ID: Final = "resp_YWJjZA==" +BACKGROUND_COST_POLL_KWARGS: Final = { + "call_type": "aget_responses", + "litellm_params": { + "metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}, + }, +} +PASSTHROUGH_SETTLEMENT_KWARGS: Final = { + "call_type": "pass_through_endpoint", + "litellm_params": {"metadata": {}}, + SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES, +} + + +def _background_create_cost_and_other_events( + cost_kwargs: dict[str, object], +) -> tuple[tuple[dict[str, object], StandardLoggingPayload], ...]: + create: Final = StandardLoggingPayload( + id=BACKGROUND_RESPONSE_ID, + call_type="aresponses", + messages=[{"role": "user", "content": "background create input"}], + ) + cost: Final = StandardLoggingPayload(id=BACKGROUND_RESPONSE_ID, call_type="aget_responses", response_cost=0.42) + ordinary: Final = StandardLoggingPayload(id="chatcmpl-ordinary", call_type="acompletion") + batch_retrieve: Final = StandardLoggingPayload(id="batch_abc123", call_type="aretrieve_batch", response_cost=0.07) + return ( + ({"call_type": "aresponses", "litellm_params": {"metadata": {}}, "standard_logging_object": create}, create), + ({**cost_kwargs, "standard_logging_object": cost}, cost), + ( + {"call_type": "acompletion", "litellm_params": {"metadata": {}}, "standard_logging_object": ordinary}, + ordinary, + ), + ( + { + "call_type": "aretrieve_batch", + "litellm_params": {"metadata": {}}, + "standard_logging_object": batch_retrieve, + }, + batch_retrieve, + ), + ) + + +def _expected_files( + events: tuple[tuple[dict[str, object], StandardLoggingPayload], ...], +) -> dict[str, StandardLoggingPayload]: + create, cost, ordinary, batch_retrieve = (payload for _, payload in events) + return { + "resp_YWJjZA.json": create, + "resp_YWJjZA_background_cost.json": cost, + "chatcmpl-ordinary.json": ordinary, + "batch_abc123.json": batch_retrieve, + } + + +async def _log_and_flush( + logger: AzureBlobStorageLogger, events: tuple[tuple[dict[str, object], StandardLoggingPayload], ...] +) -> None: + for kwargs, payload in events: + await logger.async_log_success_event(kwargs, {"id": payload["id"]}, None, None) + await logger.async_send_batch() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cost_kwargs", + ( + pytest.param(BACKGROUND_COST_POLL_KWARGS, id="cost-poll"), + pytest.param(PASSTHROUGH_SETTLEMENT_KWARGS, id="passthrough-settlement"), + ), +) +async def test_background_cost_event_keeps_its_own_file_beside_the_create_on_the_entra_path( + mock_env_vars, monkeypatch, cost_kwargs +): + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + events: Final = _background_create_cost_and_other_events(cost_kwargs) + with patch( # test-quality-ok: REST client is created inside the method; assert emitted requests + "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" + ) as mock_get_client: + mock_http_client = AsyncMock() + mock_http_client.put.return_value = MagicMock() + mock_http_client.patch.return_value = MagicMock() + mock_get_client.return_value = mock_http_client + + logger = AzureBlobStorageLogger() + logger.azure_auth_token = "mock-azure-ad-token" + logger.token_expiry = None + await _log_and_flush(logger, events) + + base_url: Final = "https://test-account.dfs.core.windows.net/test-container/" + appends: Final = [call for call in mock_http_client.patch.call_args_list if "action=append" in call.args[0]] + uploaded: Final = { + call.args[0].removeprefix(base_url).split("?")[0]: json.loads(call.kwargs["data"]) for call in appends + } + assert uploaded == _expected_files(events), ( + "the create, its background cost event and every other event must each keep their own file" + ) + + +@pytest.mark.asyncio +async def test_failure_event_is_uploaded_to_a_file_named_after_its_id(mock_env_vars, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + failure: Final = StandardLoggingPayload(id="chatcmpl-failed", call_type="acompletion", error_str="provider failure") + with patch( # test-quality-ok: REST client is created inside the method; assert emitted requests + "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" + ) as mock_get_client: + mock_http_client = AsyncMock() + mock_http_client.put.return_value = MagicMock() + mock_http_client.patch.return_value = MagicMock() + mock_get_client.return_value = mock_http_client + + logger = AzureBlobStorageLogger() + logger.azure_auth_token = "mock-azure-ad-token" + logger.token_expiry = None + await logger.async_log_failure_event( + {"call_type": "acompletion", "litellm_params": {"metadata": {}}, "standard_logging_object": failure}, + None, + None, + None, + ) + await logger.async_send_batch() + + base_url: Final = "https://test-account.dfs.core.windows.net/test-container/" + appends: Final = [call for call in mock_http_client.patch.call_args_list if "action=append" in call.args[0]] + assert {call.args[0].removeprefix(base_url).split("?")[0]: json.loads(call.kwargs["data"]) for call in appends} == { + "chatcmpl-failed.json": failure + } + + +@pytest.mark.asyncio +async def test_background_cost_event_keeps_its_own_file_beside_the_create_with_an_account_key( + workload_identity_env_vars, monkeypatch +): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdC1rZXk=") + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + monkeypatch.setattr(azure_storage_module, "datetime", _FrozenDatetime) + events: Final = _background_create_cost_and_other_events(BACKGROUND_COST_POLL_KWARGS) + + uploaded: Final[dict[str, StandardLoggingPayload]] = {} + + def file_client_for(file_path: str) -> MagicMock: + file_client = MagicMock() + file_client.create_file = AsyncMock() + file_client.flush_data = AsyncMock() + + async def append_data(data: bytes, offset: int, length: int) -> None: + uploaded[file_path] = json.loads(data) + + file_client.append_data = append_data + return file_client + + file_system_client = MagicMock() + file_system_client.get_file_client = MagicMock(side_effect=file_client_for) + service_client = MagicMock() + service_client.get_file_system_client = MagicMock(return_value=file_system_client) + fake_aio_module = MagicMock() + fake_aio_module.DataLakeServiceClient = MagicMock(return_value=service_client) + + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): + logger = AzureBlobStorageLogger() + await _log_and_flush(logger, events) + + today: Final = _FrozenDatetime.now().strftime("%Y-%m-%d") + assert uploaded == {f"{today}/{name}": payload for name, payload in _expected_files(events).items()}, ( + "the create, its background cost event and every other event must each keep their own file" + ) diff --git a/tests/unit/integrations/clickhouse/test_clickhouse_spend_logger.py b/tests/unit/integrations/clickhouse/test_clickhouse_spend_logger.py index f94d7bbee37..42e69f95bb4 100644 --- a/tests/unit/integrations/clickhouse/test_clickhouse_spend_logger.py +++ b/tests/unit/integrations/clickhouse/test_clickhouse_spend_logger.py @@ -20,10 +20,16 @@ from litellm.integrations.clickhouse.clickhouse_spend_logger import ( strip_cache_hit_suffix, ) from litellm.integrations.clickhouse.context import lens_analysis +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.internal_call_metadata import BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.secret_redaction import REDACTED from litellm.tracing.types import SpendLogRecord -from litellm.types.utils import StandardLoggingPayload +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, StandardLoggingPayload _JSON_OBJECT_ADAPTER: Final = TypeAdapter(Mapping[str, JsonValue]) @@ -349,6 +355,41 @@ def test_cache_hit_id_is_stripped_for_response_id(): assert strip_cache_hit_suffix("chatcmpl-xyz") == "chatcmpl-xyz" +@pytest.mark.parametrize( + "cost_kwargs", + ( + pytest.param( + { + "call_type": "aget_responses", + "litellm_params": { + "metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN} + }, + }, + id="cost-poll", + ), + pytest.param( + { + "call_type": "pass_through_endpoint", + "litellm_params": {"metadata": {}}, + SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES, + }, + id="passthrough-settlement", + ), + ), +) +def test_background_cost_row_keeps_its_own_request_id_beside_its_create(cost_kwargs: Mapping[str, object]) -> None: + payload: Final = cast(StandardLoggingPayload, _payload(id="resp_YWJjZA", call_type="aresponses")) + create_kwargs: Final[Mapping[str, object]] = {"call_type": "aresponses", "litellm_params": {"metadata": {}}} + + rows: Final = [spend_log_row_from_payload(payload, kwargs) for kwargs in (create_kwargs, cost_kwargs)] + + assert [row["request_id"] for row in rows] == [ + "resp_YWJjZA", + f"resp_YWJjZA{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}", + ] + assert [row["response_id"] for row in rows] == ["resp_YWJjZA", "resp_YWJjZA"] + + def test_parse_traceparent_valid_missing_malformed(): assert parse_traceparent(TRACEPARENT) == (TRACE_ID, SPAN_ID) assert parse_traceparent(None) == ("", "") diff --git a/tests/unit/integrations/conftest.py b/tests/unit/integrations/conftest.py index 48a01ed8d48..9278a280fe7 100644 --- a/tests/unit/integrations/conftest.py +++ b/tests/unit/integrations/conftest.py @@ -1,14 +1,18 @@ import functools import http.server +import importlib.abc import ipaddress import os import queue import ssl +import sys import threading -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from importlib.machinery import ModuleSpec from pathlib import Path +from types import ModuleType from typing import Final import pytest @@ -75,6 +79,36 @@ def write_self_signed_cert(directory: Path, stem: str) -> tuple[Path, Path]: return certificate_path, key_path +PROXY_ONLY_PACKAGES: Final = frozenset({"fastapi", "starlette"}) + + +class _ProxyOnlyPackageBlocker(importlib.abc.MetaPathFinder): + def find_spec( + self, + fullname: str, + path: Sequence[str] | None = None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + if fullname.partition(".")[0] in PROXY_ONLY_PACKAGES: + raise ModuleNotFoundError(f"No module named {fullname!r}") + return None + + +def _needs_proxy_extras(module_name: str) -> bool: + return ( + module_name.partition(".")[0] in PROXY_ONLY_PACKAGES + or module_name == "litellm.proxy" + or module_name.startswith("litellm.proxy.") + ) + + +@pytest.fixture +def sdk_only_install(monkeypatch: pytest.MonkeyPatch) -> None: + for module_name in tuple(filter(_needs_proxy_extras, sys.modules)): + monkeypatch.delitem(sys.modules, module_name) + monkeypatch.setattr(sys, "meta_path", [_ProxyOnlyPackageBlocker(), *sys.meta_path]) + + @pytest.fixture(autouse=True) def restore_process_environment() -> Iterator[None]: original: Final = dict(os.environ) diff --git a/tests/unit/integrations/gcs_bucket/test_gcs_bucket.py b/tests/unit/integrations/gcs_bucket/test_gcs_bucket.py index 6f143a244f6..c29985b1e41 100644 --- a/tests/unit/integrations/gcs_bucket/test_gcs_bucket.py +++ b/tests/unit/integrations/gcs_bucket/test_gcs_bucket.py @@ -13,9 +13,11 @@ import litellm import pytest import respx +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN mock_response_data: Final = { @@ -172,9 +174,7 @@ async def test_get_payload_not_found(monkeypatch): assert payload is None -def _gcs_logger_with_upload_status( - monkeypatch: pytest.MonkeyPatch, status_code: int -) -> tuple[GCSBucketLogger, Mock]: +def _gcs_logger_with_upload_status(monkeypatch: pytest.MonkeyPatch, status_code: int) -> tuple[GCSBucketLogger, Mock]: monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) monkeypatch.setattr(google.auth, "default", _google_default_credentials) monkeypatch.setenv("GCS_FLUSH_INTERVAL", "3600") @@ -394,3 +394,98 @@ async def test_batch_line_items_are_serialized_off_the_event_loop(monkeypatch: p (thread,) = threads assert thread is not threading.main_thread() + + +@pytest.mark.asyncio +async def test_unbatched_background_cost_event_keeps_its_own_object_readable_by_its_spend_log_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GCS_USE_BATCHED_LOGGING", "false") + logger, _ = _gcs_logger_with_upload_status(monkeypatch, 200) + object_prefix: Final = "/storage/v1/b/test-bucket/o/" + stored: Final[dict[str, object]] = {} + + def bucket(request: httpx.Request) -> httpx.Response: + if request.method == "POST": + stored[request.url.params["name"]] = json.loads(request.content) + return httpx.Response(200, json={"name": request.url.params["name"]}) + name: Final = request.url.path.removeprefix(object_prefix) + if name not in stored: + return httpx.Response(404, text="No such object") + return httpx.Response(200, content=json.dumps(stored[name]).encode("utf-8")) + + logger.async_httpx_client = AsyncHTTPHandler(transport=httpx.MockTransport(bucket)) + response_id: Final = "resp_YmFja2dyb3VuZA" + create: Final = { + "id": response_id, + "messages": [{"role": "user", "content": "background create input"}], + "error_str": None, + } + cost: Final = {"id": response_id, "response_cost": 0.42, "error_str": None} + ordinary: Final = {"id": "chatcmpl-ordinary", "error_str": None} + batch_retrieve: Final = {"id": "batch_abc123", "response_cost": 0.07, "error_str": None} + events: Final = ( + ({"call_type": "aresponses", "litellm_params": {"metadata": {}}}, create), + ( + { + "call_type": "aget_responses", + "litellm_params": { + "metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN} + }, + }, + cost, + ), + ({"call_type": "acompletion", "litellm_params": {"metadata": {}}}, ordinary), + ({"call_type": "aretrieve_batch", "litellm_params": {"metadata": {}}}, batch_retrieve), + ) + + for kwargs, payload in events: + await logger.async_log_success_event( + {**kwargs, "standard_logging_object": payload}, {"id": payload["id"]}, None, None + ) + await logger.flush_queue() + + assert {name.split("/", maxsplit=1)[1]: payload for name, payload in stored.items()} == { + response_id: create, + f"{response_id}_background_cost": cost, + "chatcmpl-ordinary": ordinary, + "batch_abc123": batch_retrieve, + }, "the create, its background cost event and every other event must each keep their own object" + now: Final = datetime.now(timezone.utc) + assert await logger.get_request_response_payload(f"{response_id}_background_cost", now, None) == cost + assert await logger.get_request_response_payload(response_id, now, None) == create + + +@pytest.mark.asyncio +async def test_unbatched_gcs_log_id_still_names_a_background_cost_event_object(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GCS_USE_BATCHED_LOGGING", "false") + logger, _ = _gcs_logger_with_upload_status(monkeypatch, 200) + stored: Final[dict[str, object]] = {} + + def bucket(request: httpx.Request) -> httpx.Response: + stored[request.url.params["name"]] = json.loads(request.content) + return httpx.Response(200, json={"name": request.url.params["name"]}) + + logger.async_httpx_client = AsyncHTTPHandler(transport=httpx.MockTransport(bucket)) + cost: Final = {"id": "resp_YmFja2dyb3VuZA", "response_cost": 0.42, "error_str": None} + await logger.async_log_success_event( + { + "call_type": "aget_responses", + "litellm_params": { + "metadata": { + INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, + "gcs_log_id": "qa-run-7", + } + }, + "standard_logging_object": cost, + }, + {"id": cost["id"]}, + None, + None, + ) + await logger.flush_queue() + + ((name, payload),) = stored.items() + object_name: Final = name.split("/", maxsplit=1)[1] + assert object_name.startswith("custom-") and object_name.endswith("-qa-run-7"), object_name + assert payload == cost diff --git a/tests/unit/integrations/test_deepeval.py b/tests/unit/integrations/test_deepeval.py index 6dd3aef0a03..4523a9c61d2 100644 --- a/tests/unit/integrations/test_deepeval.py +++ b/tests/unit/integrations/test_deepeval.py @@ -2,11 +2,15 @@ import os import unittest from litellm._uuid import uuid from datetime import datetime, timezone +from typing import Final from unittest.mock import MagicMock, patch from litellm.integrations.deepeval.api import Endpoints, HttpMethods from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.deepeval.types import SpanApiType, TraceSpanApiStatus +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.internal_call_metadata import BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN class TestDeepEvalLogger(unittest.TestCase): @@ -94,6 +98,37 @@ class TestDeepEvalLogger(unittest.TestCase): self.assertEqual(llm_span["inputTokenCount"], 10) self.assertEqual(llm_span["outputTokenCount"], 20) + def test_background_cost_span_keeps_its_own_uuid_beside_its_create(self) -> None: + standard_logging_object: Final = { + "id": "resp_YWJjZA", + "trace_id": self.trace_id, + "model": self.model, + "response": {"usage": {}, "choices": [{"message": {"content": "OK"}}]}, + } + create_kwargs: Final = { + "input": self.input_str, + "call_type": "aresponses", + "litellm_params": {"metadata": {}}, + "standard_logging_object": standard_logging_object, + } + cost_kwargs: Final = { + "input": self.input_str, + "call_type": "aget_responses", + "litellm_params": { + "metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN} + }, + "standard_logging_object": standard_logging_object, + } + + for kwargs in (create_kwargs, cost_kwargs): + self.logger.log_success_event(kwargs, self.mock_response_obj, self.start_time, self.end_time) + + span_uuids: Final = [ + call.kwargs["body"]["llmSpans"][0]["uuid"] + for call in self.mock_api_instance.send_request.call_args_list + ] + self.assertEqual(span_uuids, ["resp_YWJjZA", f"resp_YWJjZA{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"]) + def test_log_failure_event(self): error_message = "This is an error." kwargs = { diff --git a/tests/unit/integrations/test_dynamodb.py b/tests/unit/integrations/test_dynamodb.py new file mode 100644 index 00000000000..d139c49a49e --- /dev/null +++ b/tests/unit/integrations/test_dynamodb.py @@ -0,0 +1,52 @@ +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Final +from unittest.mock import MagicMock + +import pytest + +import litellm +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.integrations.dynamodb import DyanmoDBLogger +from litellm.litellm_core_utils.internal_call_metadata import BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + +BACKGROUND_COST_POLL_KWARGS: Final[Mapping[str, object]] = { + "call_type": "aget_responses", + "litellm_params": {"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}}, +} +PASSTHROUGH_SETTLEMENT_KWARGS: Final[Mapping[str, object]] = { + "call_type": "pass_through_endpoint", + "litellm_params": {"metadata": {}}, + SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES, +} + + +@pytest.mark.parametrize( + "cost_kwargs", + ( + pytest.param(BACKGROUND_COST_POLL_KWARGS, id="cost-poll"), + pytest.param(PASSTHROUGH_SETTLEMENT_KWARGS, id="passthrough-settlement"), + ), +) +@pytest.mark.usefixtures("sdk_only_install") +def test_background_cost_event_keeps_its_own_item_beside_its_create( + monkeypatch: pytest.MonkeyPatch, cost_kwargs: Mapping[str, object] +) -> None: + monkeypatch.setattr(litellm, "dynamodb_table_name", "litellm-logs") + dynamodb: Final = MagicMock() + logger: Final = DyanmoDBLogger(dynamodb=dynamodb) + response_obj: Final = {"id": "resp_YWJjZA", "usage": {"total_tokens": 3}} + now: Final = datetime.now(timezone.utc) + create_kwargs: Final[Mapping[str, object]] = {"call_type": "aresponses", "litellm_params": {"metadata": {}}} + + for kwargs in (create_kwargs, cost_kwargs): + logger.log_event(kwargs, response_obj, now, now, print_verbose=print) + + dynamodb.Table.assert_called_with("litellm-logs") + written_ids: Final = [call.kwargs["Item"]["id"] for call in dynamodb.Table.return_value.put_item.call_args_list] + assert written_ids == ["resp_YWJjZA", f"resp_YWJjZA{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"] diff --git a/tests/unit/integrations/test_openmeter.py b/tests/unit/integrations/test_openmeter.py index 2d09e1572db..fc4e2fcd7a3 100644 --- a/tests/unit/integrations/test_openmeter.py +++ b/tests/unit/integrations/test_openmeter.py @@ -1,11 +1,30 @@ import json import os +from collections.abc import Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest import litellm +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.openmeter import OpenMeterLogger +from litellm.litellm_core_utils.internal_call_metadata import BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + +BACKGROUND_COST_POLL_KWARGS: Final[Mapping[str, object]] = { + "call_type": "aget_responses", + "litellm_params": {"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}}, +} +PASSTHROUGH_SETTLEMENT_KWARGS: Final[Mapping[str, object]] = { + "call_type": "pass_through_endpoint", + "litellm_params": {"metadata": {}}, + SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES, +} class TestOpenMeterIntegration: @@ -36,6 +55,31 @@ class TestOpenMeterIntegration: with pytest.raises(Exception, match=r"Missing keys.*OPENMETER_API_KEY"): OpenMeterLogger() + @pytest.mark.parametrize( + "cost_kwargs", + ( + pytest.param(BACKGROUND_COST_POLL_KWARGS, id="cost-poll"), + pytest.param(PASSTHROUGH_SETTLEMENT_KWARGS, id="passthrough-settlement"), + ), + ) + @pytest.mark.usefixtures("sdk_only_install") + def test_background_cost_event_gets_its_own_event_id_beside_its_create( + self, cost_kwargs: Mapping[str, object] + ) -> None: + logger: Final = OpenMeterLogger() + response_obj: Final = {"id": "resp_YWJjZA"} + create_kwargs: Final[Mapping[str, object]] = { + "user": "user-1", + "call_type": "aresponses", + "litellm_params": {"metadata": {}}, + } + + create_event: Final = logger._common_logic(create_kwargs, response_obj) + cost_event: Final = logger._common_logic({**cost_kwargs, "user": "user-1", "response_cost": 0.42}, response_obj) + + assert create_event["id"] == "resp_YWJjZA" + assert cost_event["id"] == f"resp_YWJjZA{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}" + def test_common_logic_with_string_user(self): """Test that _common_logic correctly handles string user parameter""" logger = OpenMeterLogger() diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index f1854563cd5..13e181b2b92 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11090,6 +11090,40 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() assert logging_obj.response_cost_calculator(result=assembled) == 0.0042 +@pytest.mark.parametrize("continued_response_id", ["resp_continuedafteracutstream", None]) +@pytest.mark.parametrize("client_facing_response_id", ["resp_encryptedidtheclientsaw", None]) +def test_sync_success_handler_logs_a_stream_under_the_id_the_client_saw( + client_facing_response_id, continued_response_id +): + class Recorder(CustomLogger): + def __init__(self): + super().__init__() + self.logged = [] + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self.logged.append((response_obj.id, kwargs["standard_logging_object"]["id"])) + + recorder: Final = Recorder() + logging_obj: Final = _responses_stream_logging_obj() + logging_obj.dynamic_success_callbacks = [recorder] + logging_obj.client_facing_response_id = client_facing_response_id + if continued_response_id is not None: + logging_obj.record_continued_partial( + Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), 0.0, continued_response_id + ) + now: Final = datetime.datetime.now() + + logging_obj.success_handler( + result=_completed_responses_event(ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5)), + start_time=now, + end_time=now, + cache_hit=False, + ) + + expected_id: Final = client_facing_response_id or continued_response_id or "resp-1" + assert recorder.logged == [(expected_id, expected_id)] + + def test_response_cost_calculator_prices_terminal_responses_event_from_its_response(): logging_obj: Final = _responses_stream_logging_obj() inner_response: Final = ResponsesAPIResponse( diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index f52f40ad55d..f73d33cc93e 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -5150,6 +5150,23 @@ def test_a_background_response_billing_event_writes_a_row_apart_from_the_read_th assert request_ids == ("resp_shared", "resp_shared_background_cost") +@pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) +def test_a_read_or_cancel_row_carries_the_id_its_callbacks_logged(call_type: str): + row: Final = get_logging_payload( + kwargs={ + "call_type": call_type, + "model": "gpt-5.6", + "litellm_call_id": "call-read", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj={"id": "resp_logged_by_callbacks", "object": "response", "status": "completed"}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert row["request_id"] == "resp_logged_by_callbacks" + + @pytest.mark.parametrize("forged_marker", [True, BackgroundResponseSettlement.SETTLES.value]) def test_a_request_body_cannot_move_its_spend_row_onto_a_background_settlement_id(forged_marker): """A failed call logs its raw request body as kwargs, so a body field named like the settlement diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index f1d841a8e5e..9a8fa19e2e4 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1083,6 +1083,128 @@ async def test_previous_response_history_starts_at_a_compaction_item_in_a_stored ] +def _stored_response_turn( + request_id: str, call_type: str, status: str, output: list[dict[str, object]], prompt: str | None = None +) -> dict[str, object]: + return { + "request_id": request_id, + "call_type": call_type, + "session_id": "session-with-reads", + "proxy_server_request": json.dumps({} if prompt is None else {"input": [{"role": "user", "content": prompt}]}), + "response": {"id": request_id, "object": "response", "status": status, "output": output}, + } + + +def _identified_output_message(item_id: str, text: str) -> dict[str, object]: + return {**_output_message(text), "id": item_id} + + +def _replayed_roles_and_texts(session: ChatCompletionSession) -> list[tuple[object, object]]: + return [ + (role, _text_of(content)) for role, content in (_role_and_content(message) for message in session["messages"]) + ] + + +@pytest.mark.asyncio +async def test_reads_and_cancels_of_a_response_the_session_already_holds_replay_no_turns(): + answer: Final = [_identified_output_message("msg_answer", "OK")] + spend_logs: Final = ( + _stored_response_turn("resp_create", "aresponses", "completed", answer, prompt="Reply with OK"), + _stored_response_turn("resp_read", "aget_responses", "completed", answer), + _stored_response_turn("resp_streamed_read", "aget_responses", "completed", answer), + _stored_response_turn("resp_cancel", "acancel_responses", "completed", answer), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_create") + + assert _replayed_roles_and_texts(session) == [("user", "Reply with OK"), ("assistant", "OK")] + + +@pytest.mark.asyncio +async def test_a_polled_background_response_replays_its_finished_output_once(): + answer: Final = [_identified_output_message("msg_answer", "One, two, three")] + spend_logs: Final = ( + _stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to three"), + _stored_response_turn("resp_poll_running", "aget_responses", "in_progress", []), + _stored_response_turn("resp_poll_done", "aget_responses", "completed", answer), + _stored_response_turn("resp_poll_again", "aget_responses", "completed", answer), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_queued") + + assert _replayed_roles_and_texts(session) == [("user", "Count to three"), ("assistant", "One, two, three")] + + +@pytest.mark.asyncio +async def test_a_background_response_that_grows_between_polls_replays_each_output_item_once(): + first: Final = _identified_output_message("msg_first", "One") + second: Final = _identified_output_message("msg_second", "Two") + spend_logs: Final = ( + _stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to two"), + _stored_response_turn("resp_poll_partial", "aget_responses", "in_progress", [first]), + _stored_response_turn("resp_poll_done", "aget_responses", "completed", [first, second]), + _stored_response_turn("resp_poll_again", "aget_responses", "completed", [first, second]), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_queued") + + assert _replayed_roles_and_texts(session) == [("user", "Count to two"), ("assistant", "One"), ("assistant", "Two")] + + +@pytest.mark.asyncio +async def test_an_item_a_poll_caught_half_written_replays_its_finished_text_once(): + spend_logs: Final = ( + _stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to three"), + _stored_response_turn( + "resp_poll_partial", "aget_responses", "in_progress", [_identified_output_message("msg_answer", "One, t")] + ), + _stored_response_turn( + "resp_poll_done", + "aget_responses", + "completed", + [_identified_output_message("msg_answer", "One, two, three"), _identified_output_message("msg_more", "Done")], + ), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_queued") + + assert _replayed_roles_and_texts(session) == [ + ("user", "Count to three"), + ("assistant", "One, two, three"), + ("assistant", "Done"), + ] + + +@pytest.mark.asyncio +async def test_a_polled_output_item_without_an_id_still_replays_in_order(): + spend_logs: Final = ( + _stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to two"), + _stored_response_turn( + "resp_poll_done", + "aget_responses", + "completed", + [_output_message("One"), _identified_output_message("msg_two", "Two")], + ), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_queued") + + assert _replayed_roles_and_texts(session) == [("user", "Count to two"), ("assistant", "One"), ("assistant", "Two")] + + +@pytest.mark.asyncio +async def test_a_read_that_ends_before_its_streamed_create_replays_the_answer_once_after_the_prompt(): + answer: Final = [_identified_output_message("msg_answer", "OK")] + spend_logs: Final = ( + _stored_response_turn("resp_read_mid_stream", "aget_responses", "completed", answer), + _stored_response_turn("resp_create", "aresponses", "completed", answer, prompt="Say OK"), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_create") + + assert _replayed_roles_and_texts(session) == [("user", "Say OK"), ("assistant", "OK")] + + def _tool_call_ids(message: object) -> tuple[str, ...]: tool_calls: Final = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) return tuple(call["id"] if isinstance(call, dict) else call.id for call in tool_calls or ()) diff --git a/tests/unit/test_responses_id_security.py b/tests/unit/test_responses_id_security.py index 7e74bfd0f17..8dc50fa33c7 100644 --- a/tests/unit/test_responses_id_security.py +++ b/tests/unit/test_responses_id_security.py @@ -4,23 +4,33 @@ Tests for ResponsesIDSecurity hook. Tests the security hook that prevents user B from seeing response from user A. """ +import datetime +from collections.abc import Callable, Mapping from unittest.mock import MagicMock, patch import pytest from fastapi import HTTPException +from litellm.caching.dual_cache import DualCache +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.responses_id_security import ( ResponsesIDSecurity, _streams_response_ids, ) +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( GenericEvent, + ResponseAPIUsage, ResponseCompletedEvent, ResponseCreatedEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import SpecialEnums +from litellm.types.responses.main import DeleteResponseResult +from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse, SpecialEnums, Usage @pytest.fixture @@ -55,12 +65,12 @@ class TestIsEncryptedResponseId: import litellm.proxy.hooks.responses_id_security as responses_module with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt: - mock_decrypt.return_value = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_123;user_id:user-456" - - result = responses_id_security._is_encrypted_response_id( - "resp_encrypted_value" + mock_decrypt.return_value = ( + f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_123;user_id:user-456" ) + result = responses_id_security._is_encrypted_response_id("resp_encrypted_value") + assert result is True mock_decrypt.assert_called_once() @@ -88,9 +98,7 @@ class TestDecryptResponseId: with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt: mock_decrypt.return_value = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_original_123;user_id:user-456;team_id:team-789" - original_id, user_id, team_id = responses_id_security._decrypt_response_id( - "resp_encrypted_value" - ) + original_id, user_id, team_id = responses_id_security._decrypt_response_id("resp_encrypted_value") assert original_id == "resp_original_123" assert user_id == "user-456" @@ -104,9 +112,7 @@ class TestDecryptResponseId: with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt: mock_decrypt.return_value = None - original_id, user_id, team_id = responses_id_security._decrypt_response_id( - "resp_plain_value" - ) + original_id, user_id, team_id = responses_id_security._decrypt_response_id("resp_plain_value") assert original_id == "resp_plain_value" assert user_id is None @@ -116,9 +122,7 @@ class TestDecryptResponseId: class TestCheckUserAccessToResponseId: """Test check_user_access_to_response_id function""" - def test_check_user_access_same_user( - self, responses_id_security, mock_user_api_key_dict - ): + def test_check_user_access_same_user(self, responses_id_security, mock_user_api_key_dict): """Test that same user has access to their response ID""" result = responses_id_security.check_user_access_to_response_id( response_id_user_id="test-user-123", @@ -128,9 +132,7 @@ class TestCheckUserAccessToResponseId: assert result is True - def test_check_user_access_different_user_raises_exception( - self, responses_id_security, mock_user_api_key_dict - ): + def test_check_user_access_different_user_raises_exception(self, responses_id_security, mock_user_api_key_dict): """Test that different user is denied access to response ID""" with patch("litellm.proxy.proxy_server.general_settings", {}): with pytest.raises(HTTPException) as exc_info: @@ -143,9 +145,7 @@ class TestCheckUserAccessToResponseId: assert exc_info.value.status_code == 403 assert "Forbidden" in exc_info.value.detail - def test_check_user_access_different_team_raises_exception( - self, responses_id_security, mock_user_api_key_dict - ): + def test_check_user_access_different_team_raises_exception(self, responses_id_security, mock_user_api_key_dict): """Test that different team is denied access to response ID""" with patch("litellm.proxy.proxy_server.general_settings", {}): with pytest.raises(HTTPException) as exc_info: @@ -158,9 +158,7 @@ class TestCheckUserAccessToResponseId: assert exc_info.value.status_code == 403 assert "Forbidden" in exc_info.value.detail - def test_check_user_access_team_a_to_team_b_without_user_id( - self, responses_id_security - ): + def test_check_user_access_team_a_to_team_b_without_user_id(self, responses_id_security): """Test that key from team A (without user_id) cannot access response from team B (without user_id)""" # Create a mock user from team A without user_id mock_auth_team_a = MagicMock() @@ -179,9 +177,7 @@ class TestCheckUserAccessToResponseId: assert exc_info.value.status_code == 403 assert "team" in exc_info.value.detail.lower() - def test_check_user_access_team_a_to_team_b_with_user_id( - self, responses_id_security - ): + def test_check_user_access_team_a_to_team_b_with_user_id(self, responses_id_security): """Test that key from team A (without user_id) cannot access response from team B (with user_id)""" # Create a mock user from team A without user_id mock_auth_team_a = MagicMock() @@ -217,9 +213,7 @@ class TestCheckUserAccessToResponseId: assert result is True - def test_check_user_access_admin_can_access_any_response( - self, responses_id_security - ): + def test_check_user_access_admin_can_access_any_response(self, responses_id_security): """Test that proxy admin can access any response ID""" from litellm.proxy._types import LitellmUserRoles @@ -238,9 +232,7 @@ class TestCheckUserAccessToResponseId: assert result is True - def test_check_user_access_security_disabled( - self, responses_id_security, mock_user_api_key_dict - ): + def test_check_user_access_security_disabled(self, responses_id_security, mock_user_api_key_dict): """Test that when security is disabled, any user can access any response""" with patch( "litellm.proxy.proxy_server.general_settings", @@ -266,9 +258,7 @@ class TestAsyncPreCallHook: """Test pre-call hook decrypts previous_response_id for aresponses call""" data = {"previous_response_id": "resp_encrypted_value"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -284,15 +274,11 @@ class TestAsyncPreCallHook: assert result["previous_response_id"] == "resp_original_123" @pytest.mark.asyncio - async def test_async_pre_call_hook_aget_responses( - self, responses_id_security, mock_user_api_key_dict, mock_cache - ): + async def test_async_pre_call_hook_aget_responses(self, responses_id_security, mock_user_api_key_dict, mock_cache): """Test pre-call hook decrypts response_id for aget_responses call""" data = {"response_id": "resp_encrypted_456"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -308,9 +294,7 @@ class TestAsyncPreCallHook: assert result["response_id"] == "resp_original_456" @pytest.mark.asyncio - async def test_async_pre_call_hook_team_a_accessing_team_b_response( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_team_a_accessing_team_b_response(self, responses_id_security, mock_cache): """Test pre-call hook prevents team A from accessing team B response""" # Create a mock user from team A mock_auth_team_a = MagicMock() @@ -320,9 +304,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -341,9 +323,7 @@ class TestAsyncPreCallHook: assert "team" in exc_info.value.detail.lower() @pytest.mark.asyncio - async def test_async_pre_call_hook_team_a_accessing_team_b_with_user( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_team_a_accessing_team_b_with_user(self, responses_id_security, mock_cache): """Test pre-call hook prevents team A (no user) from accessing team B response (with user)""" # Create a mock user from team A without user_id mock_auth_team_a = MagicMock() @@ -353,9 +333,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b_with_user"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -375,9 +353,7 @@ class TestAsyncPreCallHook: assert "forbidden" in exc_info.value.detail.lower() @pytest.mark.asyncio - async def test_async_pre_call_hook_same_team_access( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_same_team_access(self, responses_id_security, mock_cache): """Test pre-call hook allows team A to access their own team's response""" # Create a mock user from team A mock_auth_team_a = MagicMock() @@ -387,9 +363,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_a"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -405,9 +379,7 @@ class TestAsyncPreCallHook: assert result["response_id"] == "resp_original_team_a" @pytest.mark.asyncio - async def test_async_pre_call_hook_adelete_responses_team_security( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_adelete_responses_team_security(self, responses_id_security, mock_cache): """Test pre-call hook prevents team A from deleting team B's response""" # Create a mock user from team A mock_auth_team_a = MagicMock() @@ -417,9 +389,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -438,9 +408,7 @@ class TestAsyncPreCallHook: assert "team" in exc_info.value.detail.lower() @pytest.mark.asyncio - async def test_async_pre_call_hook_acancel_responses_team_security( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_acancel_responses_team_security(self, responses_id_security, mock_cache): """Test pre-call hook prevents team A from canceling team B's response""" # Create a mock user from team A mock_auth_team_a = MagicMock() @@ -450,9 +418,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -470,16 +436,13 @@ class TestAsyncPreCallHook: assert exc_info.value.status_code == 403 assert "team" in exc_info.value.detail.lower() - @pytest.mark.asyncio async def test_async_pre_call_hook_alist_input_items_decrypts_response_id( self, responses_id_security, mock_user_api_key_dict, mock_cache ): data = {"response_id": "resp_encrypted_789"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -496,9 +459,7 @@ class TestAsyncPreCallHook: assert result["response_id"] == "resp_original_789" @pytest.mark.asyncio - async def test_async_pre_call_hook_alist_input_items_team_security( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_alist_input_items_team_security(self, responses_id_security, mock_cache): mock_auth_team_a = MagicMock() mock_auth_team_a.user_id = None mock_auth_team_a.team_id = "team-a" @@ -506,9 +467,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -620,9 +579,7 @@ class TestAsyncPostCallStreamingIteratorHook: "/openai/v1/responses/resp_rawprovider123", ], ) - async def test_streamed_id_encrypted_on_all_responses_routes( - self, responses_id_security, route, monkeypatch - ): + async def test_streamed_id_encrypted_on_all_responses_routes(self, responses_id_security, route, monkeypatch): streamed_id = await self._drain_streamed_id(responses_id_security, route, monkeypatch) assert streamed_id != "resp_rawprovider123" @@ -634,12 +591,8 @@ class TestAsyncPostCallStreamingIteratorHook: ) @pytest.mark.asyncio - async def test_streamed_id_untouched_on_non_responses_route( - self, responses_id_security, monkeypatch - ): - streamed_id = await self._drain_streamed_id( - responses_id_security, "/chat/completions", monkeypatch - ) + async def test_streamed_id_untouched_on_non_responses_route(self, responses_id_security, monkeypatch): + streamed_id = await self._drain_streamed_id(responses_id_security, "/chat/completions", monkeypatch) assert streamed_id == "resp_rawprovider123" assert not responses_id_security._is_encrypted_response_id(streamed_id) @@ -698,12 +651,8 @@ class TestStreamedGenericEventIdEncryption: def _advertised_ids(events): nested = (getattr(event, "response", None) for event in events) return [ - payload["id"] if isinstance(payload, dict) else payload.id - for payload in nested - if payload is not None - ] + [ - event.id for event in events if isinstance(getattr(event, "id", None), str) - ] + payload["id"] if isinstance(payload, dict) else payload.id for payload in nested if payload is not None + ] + [event.id for event in events if isinstance(getattr(event, "id", None), str)] async def _drain(self, responses_id_security, monkeypatch): monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-abcdefghij") @@ -723,9 +672,7 @@ class TestStreamedGenericEventIdEncryption: ] @pytest.mark.asyncio - async def test_every_event_advertises_the_same_encrypted_id( - self, responses_id_security, monkeypatch - ): + async def test_every_event_advertises_the_same_encrypted_id(self, responses_id_security, monkeypatch): events = await self._drain(responses_id_security, monkeypatch) advertised = self._advertised_ids(events) @@ -742,22 +689,14 @@ class TestStreamedGenericEventIdEncryption: ) @pytest.mark.asyncio - async def test_raw_provider_id_never_reaches_the_client( - self, responses_id_security, monkeypatch - ): + async def test_raw_provider_id_never_reaches_the_client(self, responses_id_security, monkeypatch): events = await self._drain(responses_id_security, monkeypatch) - assert [self.RAW_ID in event.model_dump_json() for event in events] == [ - False - ] * len(events) + assert [self.RAW_ID in event.model_dump_json() for event in events] == [False] * len(events) @pytest.mark.asyncio - async def test_sibling_fields_survive_the_rewrite( - self, responses_id_security, monkeypatch - ): - _, queued, keepalive, later, _ = await self._drain( - responses_id_security, monkeypatch - ) + async def test_sibling_fields_survive_the_rewrite(self, responses_id_security, monkeypatch): + _, queued, keepalive, later, _ = await self._drain(responses_id_security, monkeypatch) assert queued.response["status"] == "queued" assert later.response["status"] == "in_progress" @@ -769,27 +708,19 @@ class TestAsyncPostCallSuccessHook: """Test async_post_call_success_hook function""" @pytest.mark.asyncio - async def test_async_post_call_success_hook_encrypts_response( - self, responses_id_security, mock_user_api_key_dict - ): + async def test_async_post_call_success_hook_encrypts_response(self, responses_id_security, mock_user_api_key_dict): """Test post-call hook encrypts ResponsesAPIResponse""" - mock_response = ResponsesAPIResponse( - id="resp_789", created_at=1234567890, output=[], status="completed" - ) + mock_response = ResponsesAPIResponse(id="resp_789", created_at=1234567890, output=[], status="completed") data = {} - with patch.object( - responses_id_security, "_encrypt_response_id", return_value=mock_response - ) as mock_encrypt: + with patch.object(responses_id_security, "_encrypt_response_id", return_value=mock_response) as mock_encrypt: result = await responses_id_security.async_post_call_success_hook( data=data, user_api_key_dict=mock_user_api_key_dict, response=mock_response, ) - mock_encrypt.assert_called_once_with( - mock_response, mock_user_api_key_dict, request_cache=None - ) + mock_encrypt.assert_called_once_with(mock_response, mock_user_api_key_dict, request_cache=None) assert result == mock_response @pytest.mark.asyncio @@ -846,9 +777,7 @@ def _auth(user_id="owner-user", team_id="owner-team", user_role=None): def _issue_managed_id(hook, owner, provider_response_id=_FABRICATED_PROVIDER_RESPONSE_ID): """Mint an id exactly the way the proxy hands one to a client on create.""" issued = hook._encrypt_response_id( - ResponsesAPIResponse( - id=provider_response_id, created_at=1234567890, output=[], status="completed" - ), + ResponsesAPIResponse(id=provider_response_id, created_at=1234567890, output=[], status="completed"), owner, ) return issued.id @@ -1080,9 +1009,7 @@ class TestClientSuppliedRetainedIdCannotBypassAuthorization: assert exc_info.value.status_code == 403 @pytest.mark.asyncio - async def test_stranger_forging_their_own_id_never_reaches_someone_elses_response( - self, mock_cache, salt_key_env - ): + async def test_stranger_forging_their_own_id_never_reaches_someone_elses_response(self, mock_cache, salt_key_env): hook = _hook() stranger = _auth(user_id="stranger-user", team_id="stranger-team") stranger_id = _issue_managed_id(hook, stranger, provider_response_id="resp_strangerownprovideridcccccccc") @@ -1098,3 +1025,436 @@ class TestClientSuppliedRetainedIdCannotBypassAuthorization: assert result["response_id"] == "resp_strangerownprovideridcccccccc" assert result["response_id"] != victim_provider_id + + +def _admin(): + from litellm.proxy._types import LitellmUserRoles + + return _auth(user_id="admin-user", team_id="admin-team", user_role=LitellmUserRoles.PROXY_ADMIN) + + +def _provider_answer(response_id: str) -> ResponsesAPIResponse: + return ResponsesAPIResponse(id=response_id, created_at=1234567890, output=[], status="completed") + + +class TestReadsAnswerWithTheIdTheClientHolds: + """A follow-up looks its history up by the id the client holds, so a read must not hand out a new one.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) + @pytest.mark.parametrize( + "issuer, reader", + [ + (_auth(), _auth()), + (_auth(team_id=None), _auth(team_id="another-team")), + (_auth(user_id=None), _auth(user_id=None)), + (_auth(user_id=None), _auth(user_id="teammate-user")), + (_auth(), _admin()), + (_auth(user_id=None), _auth(user_id=None, team_id="admin-team", user_role=_admin().user_role)), + ], + ids=[ + "same-key", + "same-user-new-team", + "same-team-no-user", + "teammate-reading-a-team-key-id", + "admin-user", + "admin-without-user", + ], + ) + async def test_a_caller_with_access_gets_back_the_exact_id_they_sent( + self, mock_cache, salt_key_env, call_type, issuer, reader + ): + hook = _hook() + issued_id = _issue_managed_id(hook, issuer) + data = await hook.async_pre_call_hook( + user_api_key_dict=reader, cache=mock_cache, data={"response_id": issued_id}, call_type=call_type + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=reader, response=_provider_answer(data["response_id"]) + ) + + assert result.id == issued_id + + @pytest.mark.asyncio + async def test_a_follow_up_gets_an_id_for_its_own_response_never_the_previous_one(self, mock_cache, salt_key_env): + hook = _hook() + owner = _auth() + previous_id = _issue_managed_id(hook, owner) + data = await hook.async_pre_call_hook( + user_api_key_dict=owner, + cache=mock_cache, + data={"model": "gpt-fake", "previous_response_id": previous_id}, + call_type="aresponses", + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=owner, response=_provider_answer("resp_followupprovideridzzzzzzzzzzzzzzz") + ) + + assert result.id != previous_id + assert hook._decrypt_response_id(result.id) == ( + "resp_followupprovideridzzzzzzzzzzzzzzz", + "owner-user", + "owner-team", + ) + + @pytest.mark.asyncio + async def test_an_admin_read_by_a_raw_provider_id_gets_a_managed_id_back(self, mock_cache, salt_key_env): + hook = _hook() + admin = _admin() + data = await hook.async_pre_call_hook( + user_api_key_dict=admin, + cache=mock_cache, + data={"response_id": _FABRICATED_UNMANAGED_ID}, + call_type="aget_responses", + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=admin, response=_provider_answer(_FABRICATED_UNMANAGED_ID) + ) + + assert hook._is_encrypted_response_id(result.id) + assert hook._decrypt_response_id(result.id) == (_FABRICATED_UNMANAGED_ID, "admin-user", "admin-team") + + +_LOGGED_MANAGED_ID = "resp_managedidlitellmbuiltbeforethehookran" + + +def _streamed_response(response_id): + return ResponsesAPIResponse( + id=response_id, + created_at=0, + model="gpt-5.1", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5), + ) + + +def _upstream_frames(): + return ( + ResponseCreatedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_CREATED, response=_streamed_response(_LOGGED_MANAGED_ID) + ), + ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_streamed_response(_LOGGED_MANAGED_ID) + ), + ) + + +def _native_logged_result(): + return ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_streamed_response(_LOGGED_MANAGED_ID) + ) + + +def _bridged_logged_result(): + return ModelResponse( + id="chatcmpl-bridgedchatcompletionid", + model="gpt-5.1", + choices=[Choices(message=Message(role="assistant", content="hi"), finish_reason="stop", index=0)], + usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5), + ) + + +class _SpendRowRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.spend_row_ids = [] + self.standard_logging_ids = [] + self.callback_response_ids = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.spend_row_ids.append(get_logging_payload(kwargs, response_obj, start_time, end_time)["request_id"]) + self.callback_response_ids.append(response_obj.id) + if "standard_logging_object" in kwargs: + self.standard_logging_ids.append(kwargs["standard_logging_object"]["id"]) + + +async def _stream_logged_mid_stream( + hook: ResponsesIDSecurity, + route: str, + logged_result: ResponseCompletedEvent | ModelResponse, + caller: UserAPIKeyAuth | None = None, + request_data: Mapping[str, object] | None = None, +) -> tuple[str, _SpendRowRecorder]: + streaming_caller = caller if caller is not None else _auth() + recorder = _SpendRowRecorder() + logging_obj = LiteLLMLoggingObj( + model="gpt-5.1", + messages=[], + stream=True, + call_type="aresponses", + start_time=datetime.datetime.now(), + litellm_call_id="streamed-responses-call", + function_id="streamed-responses-fn", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="gpt-5.1", user="", optional_params={}, litellm_params={"api_base": ""} + ) + upstream = _upstream_frames() + + async def frames(): + for frame in upstream: + yield frame + + client_stream = hook.async_post_call_streaming_iterator_hook( + user_api_key_dict=streaming_caller.model_copy(update={"request_route": route}), + response=frames(), + request_data={**(request_data or {}), "litellm_logging_obj": logging_obj}, + ) + first_frame = await anext(client_stream) + now = datetime.datetime.now() + await logging_obj.async_success_handler(result=logged_result, start_time=now, end_time=now, cache_hit=False) + client_frames = [first_frame, *[frame async for frame in client_stream]] + return client_frames[-1].response.id, recorder + + +class TestStreamedSpendRowCarriesTheIdTheClientSaw: + @pytest.mark.asyncio + @pytest.mark.parametrize("logged_result", [_native_logged_result, _bridged_logged_result], ids=["native", "bridge"]) + @pytest.mark.parametrize("route", ["/v1/responses", "/responses", "/openai/v1/responses"]) + async def test_spend_row_and_callbacks_log_the_encrypted_id_of_the_terminal_frame( + self, salt_key_env, route, logged_result + ): + hook = _hook() + + client_id, recorder = await _stream_logged_mid_stream(hook, route, logged_result()) + + assert hook._decrypt_response_id(client_id) == (_LOGGED_MANAGED_ID, "owner-user", "owner-team") + assert recorder.spend_row_ids == [client_id] + assert recorder.standard_logging_ids == [client_id] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "hook, route", + [ + (_hook(), "/chat/completions"), + (_hook(general_settings={"disable_responses_id_security": True}), "/v1/responses"), + (_hook(signing_key=None), "/v1/responses"), + ], + ids=["not-a-create-route", "security-disabled", "no-signing-key"], + ) + async def test_stream_the_hook_leaves_alone_keeps_its_logged_id(self, salt_key_env, hook, route): + client_id, recorder = await _stream_logged_mid_stream(hook, route, _native_logged_result()) + + assert client_id == _LOGGED_MANAGED_ID + assert recorder.spend_row_ids == [_LOGGED_MANAGED_ID] + assert recorder.standard_logging_ids == [_LOGGED_MANAGED_ID] + + +class TestStreamedReadAnswersWithTheIdTheClientHolds: + @pytest.mark.asyncio + @pytest.mark.parametrize("route_prefix", ["/v1/responses", "/responses", "/openai/v1/responses"]) + async def test_the_owner_sees_their_id_while_callbacks_log_one_of_the_reads_own( + self, mock_cache, salt_key_env, route_prefix + ): + hook = _hook() + owner = _auth() + issued_id = _issue_managed_id(hook, owner, provider_response_id=_LOGGED_MANAGED_ID) + data = await hook.async_pre_call_hook( + user_api_key_dict=owner, cache=mock_cache, data={"response_id": issued_id}, call_type="aget_responses" + ) + + client_id, recorder = await _stream_logged_mid_stream( + hook, f"{route_prefix}/{issued_id}", _native_logged_result(), caller=owner, request_data=data + ) + + assert client_id == issued_id + assert recorder.spend_row_ids == recorder.standard_logging_ids + assert len(recorder.standard_logging_ids) == 1 + assert recorder.standard_logging_ids[0] != issued_id + assert hook._decrypt_response_id(recorder.standard_logging_ids[0]) == ( + _LOGGED_MANAGED_ID, + "owner-user", + "owner-team", + ) + + @pytest.mark.asyncio + async def test_an_admin_streamed_read_of_someone_elses_response_sees_the_id_sent_and_logs_one_of_its_own( + self, mock_cache, salt_key_env + ): + hook = _hook() + admin = _admin() + issued_id = _issue_managed_id(hook, _auth(), provider_response_id=_LOGGED_MANAGED_ID) + data = await hook.async_pre_call_hook( + user_api_key_dict=admin, cache=mock_cache, data={"response_id": issued_id}, call_type="aget_responses" + ) + + client_id, recorder = await _stream_logged_mid_stream( + hook, f"/v1/responses/{issued_id}", _native_logged_result(), caller=admin, request_data=data + ) + + assert client_id == issued_id + assert recorder.spend_row_ids == recorder.standard_logging_ids + assert len(recorder.standard_logging_ids) == 1 + assert hook._decrypt_response_id(recorder.standard_logging_ids[0]) == ( + _LOGGED_MANAGED_ID, + "admin-user", + "admin-team", + ) + + + @pytest.mark.asyncio + async def test_a_frame_naming_the_held_response_under_another_wrapping_still_gets_the_id_sent( + self, mock_cache, salt_key_env + ): + hook = _hook() + owner = _auth() + issued_id = _issue_managed_id(hook, owner, provider_response_id=_routed_response_id(_LOGGED_MANAGED_ID)) + data = await hook.async_pre_call_hook( + user_api_key_dict=owner, cache=mock_cache, data={"response_id": issued_id}, call_type="aget_responses" + ) + + client_id, _ = await _stream_logged_mid_stream( + hook, f"/v1/responses/{issued_id}", _native_logged_result(), caller=owner, request_data=data + ) + + assert client_id == issued_id + + +async def _read_logged_by_callbacks( + hook: ResponsesIDSecurity, + caller: UserAPIKeyAuth, + issued_id: str, + call_type: CallTypesLiteral, + cache: DualCache, + answer: Callable[[str], ResponsesAPIResponse | DeleteResponseResult] = _provider_answer, +) -> tuple[str, _SpendRowRecorder]: + recorder = _SpendRowRecorder() + now = datetime.datetime.now() + logging_obj = LiteLLMLoggingObj( + model="gpt-5.1", + messages=[], + stream=False, + call_type=call_type, + start_time=now, + litellm_call_id=f"{call_type}-call", + function_id=f"{call_type}-fn", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="gpt-5.1", user="", optional_params={}, litellm_params={"api_base": ""} + ) + data = await hook.async_pre_call_hook( + user_api_key_dict=caller, cache=cache, data={"response_id": issued_id}, call_type=call_type + ) + provider_answer = answer(data["response_id"]) + + client_response = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=caller, response=provider_answer + ) + await logging_obj.async_success_handler(result=provider_answer, start_time=now, end_time=now, cache_hit=False) + return client_response.id, recorder + + +class TestEveryReadLogsAnIdOfItsOwn: + @pytest.mark.asyncio + @pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) + @pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"]) + async def test_callbacks_and_the_spend_row_log_a_new_id_per_read_never_the_one_the_client_holds( + self, mock_cache, salt_key_env, call_type, reader + ): + hook = _hook() + issued_id = _issue_managed_id(hook, _auth()) + + reads = [await _read_logged_by_callbacks(hook, reader, issued_id, call_type, mock_cache) for _ in range(2)] + + assert [client_id for client_id, _ in reads] == [issued_id, issued_id] + logged_ids = [id_ for _, recorder in reads for id_ in recorder.standard_logging_ids] + assert [recorder.spend_row_ids for _, recorder in reads] == [[logged_id] for logged_id in logged_ids] + assert len(set(logged_ids)) == 2 + assert issued_id not in logged_ids + assert {hook._decrypt_response_id(logged_id) for logged_id in logged_ids} == { + (_FABRICATED_PROVIDER_RESPONSE_ID, reader.user_id, reader.team_id) + } + + +_RAW_UPSTREAM_RESPONSE_ID = "resp_06de59f3upstreamiddddddddddddddddd" + + +def _routed_response_id(upstream_response_id: str) -> str: + return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( + _provider_answer(upstream_response_id), "openai", {"model_info": {"id": "deployment-1"}} + ).id + + +def _deletion_of(response_id: str) -> DeleteResponseResult: + return DeleteResponseResult( + id=ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response_id), + object="response.deleted", + deleted=True, + ) + + +class TestDeleteAnswersWithTheIdTheClientHolds: + @pytest.mark.asyncio + @pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"]) + @pytest.mark.parametrize( + "stored_id", + [_routed_response_id(_RAW_UPSTREAM_RESPONSE_ID), _RAW_UPSTREAM_RESPONSE_ID], + ids=["routed-deployment", "plain-id"], + ) + async def test_a_caller_with_access_gets_back_the_exact_id_they_sent( + self, mock_cache, salt_key_env, reader, stored_id + ): + hook = _hook() + issued_id = _issue_managed_id(hook, _auth(), provider_response_id=stored_id) + data = await hook.async_pre_call_hook( + user_api_key_dict=reader, cache=mock_cache, data={"response_id": issued_id}, call_type="adelete_responses" + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=reader, response=_deletion_of(data["response_id"]) + ) + + assert (result.id, result.object, result.deleted) == (issued_id, "response.deleted", True) + + @pytest.mark.asyncio + @pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"]) + async def test_callbacks_and_the_spend_row_log_an_id_of_its_own_never_the_one_the_client_holds( + self, mock_cache, salt_key_env, reader + ): + hook = _hook() + issued_id = _issue_managed_id( + hook, _auth(), provider_response_id=_routed_response_id(_RAW_UPSTREAM_RESPONSE_ID) + ) + + client_id, recorder = await _read_logged_by_callbacks( + hook, reader, issued_id, "adelete_responses", mock_cache, answer=_deletion_of + ) + + assert client_id == issued_id + assert recorder.spend_row_ids == recorder.callback_response_ids + assert len(recorder.callback_response_ids) == 1 + assert recorder.callback_response_ids[0] != issued_id + assert hook._decrypt_response_id(recorder.callback_response_ids[0]) == ( + _RAW_UPSTREAM_RESPONSE_ID, + reader.user_id, + reader.team_id, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "sent_id", + [_routed_response_id(_RAW_UPSTREAM_RESPONSE_ID), _RAW_UPSTREAM_RESPONSE_ID], + ids=["routed-deployment", "plain-id"], + ) + async def test_an_admin_delete_by_a_raw_provider_id_gets_the_provider_id_back_and_logs_an_id_of_its_own( + self, mock_cache, salt_key_env, sent_id + ): + hook = _hook() + + client_id, recorder = await _read_logged_by_callbacks( + hook, _admin(), sent_id, "adelete_responses", mock_cache, answer=_deletion_of + ) + + assert client_id == _RAW_UPSTREAM_RESPONSE_ID + assert [hook._decrypt_response_id(logged_id) for logged_id in recorder.callback_response_ids] == [ + (_RAW_UPSTREAM_RESPONSE_ID, "admin-user", "admin-team") + ] From cc142a819132f78b35e71f3893e9660d84ec42a4 Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Mon, 2 Nov 2026 14:17:10 -0800 Subject: [PATCH 02/15] fix(cost): add deepgram/streaming/nova-3-medical and update stale Nova-3 streaming rates (#49733) The /deepgram/v1/listen WebSocket refuses nova-3-medical with 1008 because the cost map has no streaming row for it. Deepgram's pricing page does not list Nova-3 Medical, so the new row uses the $0.0077/min streaming launch price from Deepgram's 2025-03-03 announcement deepgram/streaming/nova-3 and nova-3-multilingual were $0.0048 and $0.0058 per minute against $0.0077 and $0.0092 on deepgram.com/pricing as of 2026-11-02, so they are updated too Resolves LIT-10393 --- ...odel_prices_and_context_window_backup.json | 29 ++++++++++++++----- model_prices_and_context_window.json | 29 ++++++++++++++----- .../deepgram/test_deepgram_common_utils.py | 6 +++- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8bfb8949890..517aead5d29 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23523,12 +23523,12 @@ ] }, "deepgram/streaming/nova-3": { - "input_cost_per_second": 8e-05, + "input_cost_per_second": 0.00012833, "litellm_provider": "deepgram", "metadata": { - "calculation": "$0.0048/60 seconds = $0.00008000 per second", + "calculation": "$0.0077/60 seconds = $0.00012833 per second", "note": "Nova-3 monolingual streaming, pay as you go", - "original_pricing_per_minute": 0.0048 + "original_pricing_per_minute": 0.0077 }, "mode": "audio_transcription", "output_cost_per_second": 0.0, @@ -23537,13 +23537,28 @@ "/v1/listen" ] }, - "deepgram/streaming/nova-3-multilingual": { - "input_cost_per_second": 9.667e-05, + "deepgram/streaming/nova-3-medical": { + "input_cost_per_second": 0.00012833, "litellm_provider": "deepgram", "metadata": { - "calculation": "$0.0058/60 seconds = $0.00009667 per second", + "calculation": "$0.0077/60 seconds = $0.00012833 per second", + "note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate", + "original_pricing_per_minute": 0.0077 + }, + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://www.businesswire.com/news/home/20250303470439/en", + "supported_endpoints": [ + "/v1/listen" + ] + }, + "deepgram/streaming/nova-3-multilingual": { + "input_cost_per_second": 0.00015333, + "litellm_provider": "deepgram", + "metadata": { + "calculation": "$0.0092/60 seconds = $0.00015333 per second", "note": "Nova-3 multilingual (language=multi) streaming, pay as you go", - "original_pricing_per_minute": 0.0058 + "original_pricing_per_minute": 0.0092 }, "mode": "audio_transcription", "output_cost_per_second": 0.0, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8bfb8949890..517aead5d29 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23523,12 +23523,12 @@ ] }, "deepgram/streaming/nova-3": { - "input_cost_per_second": 8e-05, + "input_cost_per_second": 0.00012833, "litellm_provider": "deepgram", "metadata": { - "calculation": "$0.0048/60 seconds = $0.00008000 per second", + "calculation": "$0.0077/60 seconds = $0.00012833 per second", "note": "Nova-3 monolingual streaming, pay as you go", - "original_pricing_per_minute": 0.0048 + "original_pricing_per_minute": 0.0077 }, "mode": "audio_transcription", "output_cost_per_second": 0.0, @@ -23537,13 +23537,28 @@ "/v1/listen" ] }, - "deepgram/streaming/nova-3-multilingual": { - "input_cost_per_second": 9.667e-05, + "deepgram/streaming/nova-3-medical": { + "input_cost_per_second": 0.00012833, "litellm_provider": "deepgram", "metadata": { - "calculation": "$0.0058/60 seconds = $0.00009667 per second", + "calculation": "$0.0077/60 seconds = $0.00012833 per second", + "note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate", + "original_pricing_per_minute": 0.0077 + }, + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://www.businesswire.com/news/home/20250303470439/en", + "supported_endpoints": [ + "/v1/listen" + ] + }, + "deepgram/streaming/nova-3-multilingual": { + "input_cost_per_second": 0.00015333, + "litellm_provider": "deepgram", + "metadata": { + "calculation": "$0.0092/60 seconds = $0.00015333 per second", "note": "Nova-3 multilingual (language=multi) streaming, pay as you go", - "original_pricing_per_minute": 0.0058 + "original_pricing_per_minute": 0.0092 }, "mode": "audio_transcription", "output_cost_per_second": 0.0, diff --git a/tests/unit/llms/deepgram/test_deepgram_common_utils.py b/tests/unit/llms/deepgram/test_deepgram_common_utils.py index 4c1b4603076..a1672e965bb 100644 --- a/tests/unit/llms/deepgram/test_deepgram_common_utils.py +++ b/tests/unit/llms/deepgram/test_deepgram_common_utils.py @@ -285,6 +285,9 @@ NOVA_2_URL: Final = "wss://api.deepgram.com/v1/listen?model=nova-2" [ pytest.param(NOVA_3_URL, (), True, id="streaming entry present"), pytest.param(f"{NOVA_3_URL}&language=multi", (), True, id="multilingual entry present"), + pytest.param( + "wss://api.deepgram.com/v1/listen?model=nova-3-medical", (), True, id="medical streaming entry present" + ), pytest.param(NOVA_2_URL, (), False, id="only the pre-recorded entry"), pytest.param(f"{NOVA_2_URL}&language=multi", ("deepgram/streaming/nova-2",), False, id="needs multilingual"), pytest.param("wss://api.deepgram.com/v1/listen?model=nova-99-unmapped", (), False, id="nothing priced"), @@ -295,7 +298,8 @@ NOVA_2_URL: Final = "wss://api.deepgram.com/v1/listen?model=nova-2" def test_deepgram_listen_is_priced( monkeypatch: pytest.MonkeyPatch, upstream_url: str, extra_rows: tuple[str, ...], expected: bool ): - """The bundled map prices only nova-3 for streaming; nova-2 has a pre-recorded row, which must never count.""" + """The bundled map prices nova-3 and nova-3-medical for streaming; nova-2 has only a pre-recorded row, which must + never count.""" monkeypatch.delitem(litellm.model_cost, "deepgram/streaming/nova-2", raising=False) assert "deepgram/nova-2" in litellm.model_cost for row in extra_rows: From 10b3deae0d058d4da0f71d5d4642082bc5102743 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Mon, 2 Nov 2026 22:24:02 +0000 Subject: [PATCH 03/15] build(deps): allow huggingface-hub 2.x (#49056) Widen the huggingface-hub cap from <2.0 to <3.0. The locked version stays at 1.32.0; only the matching specifier line in uv.lock changes. Claude-Session: https://claude.ai/code/session_016isSPojcgVpmxSRshXQEPw Co-authored-by: Claude Opus 5.5 Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5c23a64ebb3..7a653b28f81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "tiktoken>=0.12.0,<1.0; python_version == '3.14'", "tiktoken>=0.14.0,<1.0; python_version >= '3.15'", "tokenizers>=0.21.0,<1.0", - "huggingface-hub>=0.34.0,<2.0", + "huggingface-hub>=0.34.0,<3.0", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", diff --git a/uv.lock b/uv.lock index 19d3a1ffd4f..e5948530f17 100644 --- a/uv.lock +++ b/uv.lock @@ -5262,7 +5262,7 @@ requires-dist = [ { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" }, { name = "httpx2", marker = "extra == 'mcp'", specifier = ">=2.10.0,<3" }, { name = "httpx2", marker = "extra == 'proxy'", specifier = ">=2.10.0,<3" }, - { name = "huggingface-hub", specifier = ">=0.34.0,<2.0" }, + { name = "huggingface-hub", specifier = ">=0.34.0,<3.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" }, From 523b87ec718cc733363ac0d66312730c51a23a27 Mon Sep 17 00:00:00 2001 From: Etienne Chabert Date: Mon, 2 Nov 2026 23:41:59 +0100 Subject: [PATCH 04/15] fix(proxy): charge budget windows when the cached key or team expired before the request ended (#47608) * fix(proxy): charge budget windows when the cached key or team expired before the request ended The post-call spend update reads a key's or team's budget_limits from user_api_key_cache and returned early on a miss, skipping both the window counter increment and the LiteLLM_BudgetWindowSpend row. The cached object expires after user_api_key_cache_ttl (60s by default), so any request that ran longer than that without another request refreshing the entry was counted in the key's lifetime spend but never in its budget windows. With budget reservation disabled nothing else charges the windows, so keys kept running past their window caps On a miss the update now reads budget_limits straight from the key or team row. It does not re-cache the object, since a write from the spend path could race a /key/delete and put a just-deleted key back into the auth cache * test(proxy): assert the budget window fallback never re-caches the key or team * fix(proxy): read a missing key or team row once per cache TTL in the budget window fallback The master key and UI session tokens are logged under names no key row carries, so every one of their requests missed the auth cache and read the database. A missing row is now remembered for the cache TTL in this process only. A failed read is not remembered, and a row that exists is read fresh on every miss so a key update is never served stale. * fix(proxy): skip the budget window row read for a key or team with no windows The fallback remembered only a missing row, so a key or team that exists but holds no budget_limits was read from the database on every request that outlived the key cache. It now remembers both cases under one cache key built in user_api_key_cache.py, for the same TTL as the auth cache * fix(proxy): keep no-window markers in their own cache and clear them when an update adds windows Markers for keys and teams with no budget windows lived in the shared 200-entry auth cache, so a burst of late costs could evict cached teams and users. They now live in a dedicated in-memory cache on UserApiKeyCache, reached by local and broadcast evictions. /key/update and /team/update now evict the marker on every worker, so a late cost after an update charges the windows the update added. * test(proxy): give the spend counter tests the proxy's real key cache and type the window spend helper * fix(proxy): keep a row read that an update outran from restoring the no-window marker * fix(proxy): clear finished no-window reads and evict a marker-shaped user id from both caches * fix(proxy): apply other spend counters before a window row read and let no-window notes follow the key cache settings --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/caching/dual_cache.py | 2 +- litellm/proxy/auth/auth_checks.py | 12 + .../auth_cache_invalidation_pubsub.py | 2 +- .../proxy/common_utils/user_api_key_cache.py | 22 ++ .../management_endpoints/team_endpoints.py | 4 + litellm/proxy/proxy_server.py | 257 +++++++++------ .../test_auth_cache_invalidation_pubsub.py | 29 +- .../common_utils/test_user_api_key_cache.py | 37 ++- .../test_key_management_endpoints.py | 13 +- .../test_team_endpoints.py | 8 +- tests/unit/proxy/test_budget_reservation.py | 2 +- ...test_proxy_server_endpoints_and_startup.py | 307 +++++++++++++++++- 12 files changed, 575 insertions(+), 120 deletions(-) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index e6214a3650c..c82d25c9425 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -673,7 +673,7 @@ class DualCache(BaseCache): if self.redis_cache is not None: self.redis_cache.flush_cache() - def delete_cache(self, key): + def delete_cache(self, key: str) -> None: """ Delete a key from the cache """ diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f5d3a80ee91..fdc8aa82268 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -125,6 +125,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( NO_TEAM_MEMBERSHIP_SENTINEL, TAG_REGISTRY_OVERFLOW_SENTINEL, UserApiKeyCache, + budget_windows_absent_cache_key, end_user_cache_key, end_user_restricted_registry_cache_key, get_management_object_ttl, @@ -3471,11 +3472,22 @@ async def delete_cache_key_object( ) await publish_auth_cache_invalidation(cache_key=key) + await evict_budget_windows_absent_marker( + entity_type=Litellm_EntityType.KEY, entity_id=hashed_token, user_api_key_cache=user_api_key_cache + ) _delete_cache_key_object: Final = delete_cache_key_object +async def evict_budget_windows_absent_marker( + entity_type: Litellm_EntityType, entity_id: str, user_api_key_cache: UserApiKeyCache +) -> None: + marker: Final = budget_windows_absent_cache_key(entity_type.value, entity_id) + evict_local(cache_keys=(marker,), user_api_key_cache=user_api_key_cache) + await publish_auth_cache_invalidation(cache_key=marker) + + async def delete_cache_key_objects( hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache, diff --git a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py index b2640d2a765..edc8048ee73 100644 --- a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py +++ b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py @@ -183,7 +183,7 @@ def evict_local(cache_keys: Sequence[str], user_api_key_cache: "UserApiKeyCache" def _drop_from_memory(cache_key: str, user_api_key_cache: "UserApiKeyCache") -> None: - user_api_key_cache.in_memory_cache_for(cache_key).delete_cache(cache_key) + user_api_key_cache.delete_from_memory(cache_key) @with_service_target(AUTH_OBJECTS_TARGET) diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index e64fd755ef9..a2272c3f73d 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -24,6 +24,7 @@ T = TypeVar("T", bound=BaseModel) from litellm.constants import AUTH_OBJECTS_TARGET as AUTH_OBJECTS_TARGET # noqa: E402, PLC0414 # public re-export _HASHED_TOKEN_CACHE_KEY: Final = re.compile(r"[0-9a-f]{64}") +_BUDGET_WINDOWS_ABSENT_PREFIX: Final = "budget_windows_absent:" def is_user_key_cache_key(key: str) -> bool: @@ -70,6 +71,7 @@ class UserApiKeyCache(DualCache): default_in_memory_ttl: float | None = None, default_redis_ttl: float | None = None, key_object_in_memory_cache: InMemoryCache | None = None, + budget_windows_absent_in_memory_cache: InMemoryCache | None = None, ) -> None: super().__init__( in_memory_cache=in_memory_cache, @@ -83,19 +85,33 @@ class UserApiKeyCache(DualCache): default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl, ) + self.budget_windows_absent_cache: Final = DualCache( + in_memory_cache=budget_windows_absent_in_memory_cache + or InMemoryCache(default_ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL), + default_in_memory_ttl=default_in_memory_ttl, + ) def in_memory_cache_for(self, key: str) -> InMemoryCache: return self.key_object_cache.in_memory_cache if is_user_key_cache_key(key) else self.in_memory_cache + def delete_from_memory(self, key: str) -> None: + self.in_memory_cache_for(key).delete_cache(key) + if key.startswith(_BUDGET_WINDOWS_ABSENT_PREFIX): + self.budget_windows_absent_cache.delete_cache(key) + def update_cache_ttl(self, default_in_memory_ttl: float | None, default_redis_ttl: float | None) -> None: super().update_cache_ttl(default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl) self.key_object_cache.update_cache_ttl( default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl ) + self.budget_windows_absent_cache.update_cache_ttl( + default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=None + ) def update_in_memory_max_size(self, max_size: int | None) -> None: super().update_in_memory_max_size(max_size) self.key_object_cache.update_in_memory_max_size(max_size) + self.budget_windows_absent_cache.update_in_memory_max_size(max_size) def attach_redis_cache( self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None @@ -272,6 +288,7 @@ class UserApiKeyCache(DualCache): def flush_cache(self) -> None: super().flush_cache() self.key_object_cache.in_memory_cache.flush_cache() + self.budget_windows_absent_cache.flush_cache() async def async_set_cache_pipeline( self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs: object @@ -400,6 +417,11 @@ def team_membership_reservation_cache_key(user_id: str, team_id: str) -> str: NO_TEAM_MEMBERSHIP_SENTINEL: Final = "__no_team_membership__" +def budget_windows_absent_cache_key(entity_type: str, entity_id: str) -> str: + """Cache key marking a key or team that holds no budget windows, so the spend fallback skips its row read.""" + return f"{_BUDGET_WINDOWS_ABSENT_PREFIX}{entity_type}:{entity_id}" + + def get_management_object_ttl(cache: DualCache) -> float: """ In-memory TTL for management-object cache writes (keys, teams, users, budgets, ...). diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index b712ba42a6a..f505406fe0e 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -109,6 +109,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo can_org_access_model, delete_cache_key_objects, delete_cache_team_object, + evict_budget_windows_absent_marker, get_jwt_key_mapping_cache_keys_for_tokens, get_org_object, get_team_membership, @@ -612,6 +613,9 @@ async def refresh_cached_team( await publish_auth_cache_invalidation(cache_key=f"team_id:{team_row.team_id}") if team_table.team_alias is not None: await publish_auth_cache_invalidation(cache_key=f"team_alias:{team_table.team_alias}") + await evict_budget_windows_absent_marker( + entity_type=Litellm_EntityType.TEAM, entity_id=team_row.team_id, user_api_key_cache=user_api_key_cache + ) _refresh_cached_team: Final = refresh_cached_team diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 807f9a0860d..22e5bd6b3bd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -26,6 +26,7 @@ from collections.abc import ( Collection, Container, Iterable, + Iterator, Mapping, MutableMapping, Sequence, @@ -588,6 +589,7 @@ from litellm.proxy.common_utils.token_count_log import ( ) from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, + budget_windows_absent_cache_key, end_user_cache_key, get_management_object_ttl, model_access_group_cache_key, @@ -3636,6 +3638,52 @@ def _cached_budget_limits(cached: object) -> object: return _UNTYPED_VALUE.validate_python(json.loads(stored)) if isinstance(stored, str) else stored +def _has_budget_windows(budget_limits: object) -> bool: + return isinstance(budget_limits, list) and budget_limits != [] + + +def _budget_windows_absent_marker(absent_key: str) -> object: + return _UNTYPED_VALUE.validate_python(user_api_key_cache.budget_windows_absent_cache.get_cache(key=absent_key)) + + +def _finish_pending_read(absent_key: str, pending_read: str, remember_absent: bool) -> None: + if _budget_windows_absent_marker(absent_key) != pending_read: + return + if remember_absent: + user_api_key_cache.budget_windows_absent_cache.set_cache(key=absent_key, value=True) + return + user_api_key_cache.budget_windows_absent_cache.delete_cache(absent_key) + + +async def _stored_budget_limits(entity_type: Litellm_EntityType, entity_id: str) -> object: + """Read only: caching the row here could race a delete and put a just-deleted key back into the auth cache.""" + if prisma_client is None: + return None + absent_key: Final = budget_windows_absent_cache_key(entity_type.value, entity_id) + if _budget_windows_absent_marker(absent_key) is True: + return None + pending_read: Final = f"reading:{secrets.token_hex(8)}" + user_api_key_cache.budget_windows_absent_cache.set_cache(key=absent_key, value=pending_read) + try: + row: Final[SupportsModelDump | None] = ( + await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": entity_id}) + if entity_type == Litellm_EntityType.KEY + else await TeamRepository(prisma_client).table.find_unique(where={"team_id": entity_id}) + ) + except Exception as e: # noqa: BLE001 # spend tracking must never fail the cost callback + verbose_proxy_logger.warning( + "Unable to read budget windows for %s=%s; this request's window spend is not counted: %s", + entity_type.value, + entity_id, + e, + ) + _finish_pending_read(absent_key, pending_read, remember_absent=False) + return None + budget_limits: Final[object] = None if row is None else _cached_budget_limits(row) + _finish_pending_read(absent_key, pending_read, remember_absent=not _has_budget_windows(budget_limits)) + return budget_limits + + async def _increment_spend_counters_batched( token: str | None, team_id: str | None, @@ -3666,7 +3714,68 @@ async def _increment_spend_counters_batched( cost: Final[float] = response_cost - async def _key_scope(key_token: str) -> tuple[PendingSpendIncrement | BaseException, ...]: + async def _window_increment( + entity_type: Litellm_EntityType, entity_id: str, window: object + ) -> PendingSpendIncrement | None: + duration: Final = ( + window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None) + ) + window_reset_at: Final = ( + window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None) + ) + window_counter: Final = f"spend:{entity_type.value}:{entity_id}:window:{duration}" + window_start: Final = get_budget_window_start(window) + pending_window: Final = ( + await _prepare_window_spend_counter_increment( + counter_key=window_counter, + entity_type=entity_type.value.capitalize(), + entity_id=entity_id, + window_duration=duration, + window_start=window_start, + increment=cost, + ) + if window_counter not in reserved_counter_keys + else None + ) + await _enqueue_window_spend_row_update( + entity_type=entity_type, + entity_id=entity_id, + reset_at=window_reset_at, + window_duration=duration, + window_start=window_start, + increment=cost, + request_started_at=request_started_at, + ) + return pending_window + + async def _window_increments( + entity_type: Litellm_EntityType, entity_id: str, budget_limits: object + ) -> tuple[PendingSpendIncrement | BaseException, ...]: + if not isinstance(budget_limits, list): + return () + window_pending: Final = await asyncio.gather( + *(_window_increment(entity_type, entity_id, window) for window in budget_limits), return_exceptions=True + ) + return tuple(item for item in window_pending if item is not None) + + async def _cached_window_increments( + entity_type: Litellm_EntityType, entity_id: str, cache_key: str + ) -> tuple[_SpendScopeItem, ...]: + cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key)) + if cached is None: + return (_UncachedBudgetWindows(entity_type=entity_type, entity_id=entity_id),) + return await _window_increments(entity_type, entity_id, _cached_budget_limits(cached)) + + async def _stored_window_increments( + uncached: _UncachedBudgetWindows, + ) -> tuple[PendingSpendIncrement | BaseException, ...]: + return await _window_increments( + uncached.entity_type, + uncached.entity_id, + await _stored_budget_limits(uncached.entity_type, uncached.entity_id), + ) + + async def _key_scope(key_token: str) -> tuple[_SpendScopeItem, ...]: # key_token arrives pre-hashed from metadata["user_api_key"] (auth flow # hashes raw "sk-..." keys before they reach the callback). The # startswith("sk-") check is a safety net matching update_cache — @@ -3689,52 +3798,9 @@ async def _increment_spend_counters_batched( ) ) - async def _key_window_increment(window: object) -> PendingSpendIncrement | None: - duration = ( - window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None) - ) - key_window_reset_at = ( - window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None) - ) - key_window_counter: Final = f"spend:key:{hashed_token}:window:{duration}" - key_window_start = get_budget_window_start(window) - pending_window: Final = ( - await _prepare_window_spend_counter_increment( - counter_key=key_window_counter, - entity_type="Key", - entity_id=hashed_token, - window_duration=duration, - window_start=key_window_start, - increment=cost, - ) - if key_window_counter not in reserved_counter_keys - else None - ) - await _enqueue_window_spend_row_update( - entity_type=Litellm_EntityType.KEY, - entity_id=hashed_token, - reset_at=key_window_reset_at, - window_duration=duration, - window_start=key_window_start, - increment=cost, - request_started_at=request_started_at, - ) - return pending_window + return key_pending + await _cached_window_increments(Litellm_EntityType.KEY, hashed_token, hashed_token) - key_obj: Final[object] = _UNTYPED_VALUE.validate_python( - await user_api_key_cache.async_get_cache(key=hashed_token) - ) - if key_obj is None: - return key_pending - key_budget_limits: Final = _cached_budget_limits(key_obj) - if not isinstance(key_budget_limits, list): - return key_pending - window_pending: Final = await asyncio.gather( - *(_key_window_increment(window) for window in key_budget_limits), return_exceptions=True - ) - return key_pending + tuple(item for item in window_pending if item is not None) - - async def _team_scope(scope_team_id: str) -> tuple[PendingSpendIncrement | BaseException, ...]: + async def _team_scope(scope_team_id: str) -> tuple[_SpendScopeItem, ...]: team_counter_key: Final = f"spend:team:{scope_team_id}" team_pending: Final[tuple[PendingSpendIncrement, ...]] = ( () @@ -3748,50 +3814,9 @@ async def _increment_spend_counters_batched( ) ) - async def _team_window_increment(window: object) -> PendingSpendIncrement | None: - duration = ( - window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None) - ) - team_window_reset_at = ( - window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None) - ) - team_window_counter: Final = f"spend:team:{scope_team_id}:window:{duration}" - team_window_start = get_budget_window_start(window) - pending_window: Final = ( - await _prepare_window_spend_counter_increment( - counter_key=team_window_counter, - entity_type="Team", - entity_id=scope_team_id, - window_duration=duration, - window_start=team_window_start, - increment=cost, - ) - if team_window_counter not in reserved_counter_keys - else None - ) - await _enqueue_window_spend_row_update( - entity_type=Litellm_EntityType.TEAM, - entity_id=scope_team_id, - reset_at=team_window_reset_at, - window_duration=duration, - window_start=team_window_start, - increment=cost, - request_started_at=request_started_at, - ) - return pending_window - - team_obj: Final[object] = _UNTYPED_VALUE.validate_python( - await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}") + return team_pending + await _cached_window_increments( + Litellm_EntityType.TEAM, scope_team_id, f"team_id:{scope_team_id}" ) - if team_obj is None: - return team_pending - team_budget_limits: Final = _cached_budget_limits(team_obj) - if not isinstance(team_budget_limits, list): - return team_pending - window_pending: Final = await asyncio.gather( - *(_team_window_increment(window) for window in team_budget_limits), return_exceptions=True - ) - return team_pending + tuple(item for item in window_pending if item is not None) async def _team_member_scope( scope_user_id: str, scope_team_id: str @@ -3862,22 +3887,30 @@ async def _increment_spend_counters_batched( # return_exceptions so a failing scope does not leave its siblings running # as orphaned tasks that race the caller's reservation-counter invalidation; # all scopes settle, then the first error propagates as before. - scope_results: Final = await asyncio.gather(*scope_coros, return_exceptions=True) - scope_errors: Final = tuple( - item - for scope in scope_results - for item in (scope if isinstance(scope, tuple) else (scope,)) - if isinstance(item, BaseException) + scope_items: Final = tuple(_flatten_spend_scopes(await asyncio.gather(*scope_coros, return_exceptions=True))) + uncached_windows: Final = tuple(item for item in scope_items if isinstance(item, _UncachedBudgetWindows)) + await _apply_spend_counter_increments( + pending=reservation_update.pending + + tuple(item for item in scope_items if isinstance(item, PendingSpendIncrement)) ) - pending: Final = tuple( - item - for scope in scope_results - if not isinstance(scope, BaseException) - for item in scope - if not isinstance(item, BaseException) + stored_window_items: Final = ( + tuple( + _flatten_spend_scopes( + await asyncio.gather( + *(_stored_window_increments(uncached) for uncached in uncached_windows), return_exceptions=True + ) + ) + ) + if uncached_windows + else () ) - await _apply_spend_counter_increments(pending=reservation_update.pending + pending) + stored_window_pending: Final = tuple( + item for item in stored_window_items if isinstance(item, PendingSpendIncrement) + ) + if stored_window_pending: + await _apply_spend_counter_increments(pending=stored_window_pending) stamp_budget_reservation_actual_cost(budget_reservation=budget_reservation, actual_cost=response_cost) + scope_errors: Final = tuple(item for item in scope_items + stored_window_items if isinstance(item, BaseException)) if scope_errors: raise scope_errors[0] @@ -3885,6 +3918,25 @@ async def _increment_spend_counters_batched( budget_reservation["finalized"] = True +@dataclass(frozen=True, slots=True) +class _UncachedBudgetWindows: + entity_type: Litellm_EntityType + entity_id: str + + +_SpendScopeItem: TypeAlias = PendingSpendIncrement | BaseException | _UncachedBudgetWindows + + +def _flatten_spend_scopes( + scope_results: Sequence[tuple[_SpendScopeItem, ...] | BaseException], +) -> Iterator[_SpendScopeItem]: + for scope in scope_results: + if isinstance(scope, BaseException): + yield scope + continue + yield from scope + + @dataclass(frozen=True, slots=True) class _ReservationCounterUpdate: """The reserved counters the direct increment must skip, and the adjustments that settle them on the actual @@ -14840,6 +14892,7 @@ from litellm.repositories.table_repositories import ( ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository +from litellm.repositories.verification_token_repository import VerificationTokenRepository from litellm.responses.litellm_completion_transformation.count_request import ( ChatCountBridgeError, chat_count_request, diff --git a/tests/unit/proxy/common_utils/test_auth_cache_invalidation_pubsub.py b/tests/unit/proxy/common_utils/test_auth_cache_invalidation_pubsub.py index 694b817ab86..2dc98b5ca6b 100644 --- a/tests/unit/proxy/common_utils/test_auth_cache_invalidation_pubsub.py +++ b/tests/unit/proxy/common_utils/test_auth_cache_invalidation_pubsub.py @@ -3,6 +3,7 @@ import hashlib import json import time from collections.abc import Iterable +from typing import Final from unittest.mock import patch import pytest @@ -19,7 +20,11 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( evict_and_broadcast, publish_auth_cache_invalidation, ) -from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache +from litellm.proxy.common_utils.user_api_key_cache import ( + AUTH_OBJECTS_TARGET, + UserApiKeyCache, + budget_windows_absent_cache_key, +) class _RecordingRedisClient(Redis): @@ -243,6 +248,28 @@ async def test_subscriber_deletes_key_object_partition_entry_on_message() -> Non assert cache.get_cache(hashed_token, model_type=UserAPIKeyAuth) is None +@pytest.mark.asyncio +async def test_subscriber_evicts_a_user_whose_id_looks_like_a_no_window_marker_along_with_the_marker() -> None: + user_id: Final = budget_windows_absent_cache_key("key", "alice") + cache: Final = UserApiKeyCache() + cache.set_cache(user_id, {"user_id": user_id, "models": ["gpt-4o"]}) + cache.budget_windows_absent_cache.set_cache(key=user_id, value=True) + + pubsub: Final = _QueuePubSub(initial_messages=[_invalidation_message(user_id)]) + subscriber: Final = AuthCacheInvalidationSubscriber( + redis_cache=_FakeRedisCache(client=_ScriptedPubSubRedisClient(pubsubs=[pubsub])), + user_api_key_cache=cache, + ) + subscriber.start() + try: + assert await asyncio.wait_for(pubsub.drained.wait(), timeout=5) + finally: + await subscriber.stop() + + assert cache.get_cache(user_id) is None + assert cache.budget_windows_absent_cache.get_cache(key=user_id) is None + + @pytest.mark.asyncio async def test_subscriber_deletes_additional_in_memory_cache_entry_on_message() -> None: """ diff --git a/tests/unit/proxy/common_utils/test_user_api_key_cache.py b/tests/unit/proxy/common_utils/test_user_api_key_cache.py index 09523cd1901..577e5e5a389 100644 --- a/tests/unit/proxy/common_utils/test_user_api_key_cache.py +++ b/tests/unit/proxy/common_utils/test_user_api_key_cache.py @@ -1,6 +1,6 @@ import hashlib import json -from typing import Any +from typing import Any, Final import pytest @@ -11,6 +11,7 @@ from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, + budget_windows_absent_cache_key, end_user_cache_key, get_management_object_ttl, is_user_key_cache_key, @@ -32,6 +33,17 @@ class CapturingInMemoryCache(InMemoryCache): super().set_cache(key, value, **kwargs) +class _ManualClock: + def __init__(self) -> None: + self.seconds = 1000.0 + + def __call__(self) -> float: + return self.seconds + + def advance(self, seconds: float) -> None: + self.seconds += seconds + + class FakeRedisCache(RedisCache): """ In-memory fake that enforces the UserApiKeyCache Redis payload contract. @@ -432,6 +444,29 @@ class TestUserKeyObjectPartition: assert first_key.token == tokens[0] assert cache.get_cache(end_user_cache_key("u0")) == {"user_id": "u0"} + def test_update_in_memory_max_size_applies_to_no_window_notes(self): + cache = UserApiKeyCache() + cache.update_in_memory_max_size(1) + + cache.budget_windows_absent_cache.set_cache(key=budget_windows_absent_cache_key("key", "first"), value=True) + cache.budget_windows_absent_cache.set_cache(key=budget_windows_absent_cache_key("key", "second"), value=True) + + assert cache.budget_windows_absent_cache.get_cache(key=budget_windows_absent_cache_key("key", "first")) is None + assert cache.budget_windows_absent_cache.get_cache(key=budget_windows_absent_cache_key("key", "second")) is True + + def test_update_cache_ttl_applies_to_no_window_notes(self): + clock: Final = _ManualClock() + cache: Final = UserApiKeyCache( + default_in_memory_ttl=60, budget_windows_absent_in_memory_cache=InMemoryCache(clock=clock) + ) + cache.update_cache_ttl(default_in_memory_ttl=5, default_redis_ttl=5) + note: Final = budget_windows_absent_cache_key("key", "no-windows") + + cache.budget_windows_absent_cache.set_cache(key=note, value=True) + clock.advance(6) + + assert cache.budget_windows_absent_cache.get_cache(key=note) is None + def test_update_in_memory_max_size_none_resets_key_object_partition_to_default(self): cache = UserApiKeyCache(key_object_in_memory_cache=InMemoryCache(max_size_in_memory=1)) cache.update_in_memory_max_size(None) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 0198f582be7..b453f985512 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -6,7 +6,7 @@ from contextlib import ExitStack from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, call, patch from uuid import UUID, uuid4 import pytest @@ -51,7 +51,11 @@ from litellm.proxy.auth.auth_checks import ( ) from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber -from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key +from litellm.proxy.common_utils.user_api_key_cache import ( + UserApiKeyCache, + budget_windows_absent_cache_key, + project_cache_key, +) from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker from litellm.proxy.management_endpoints.key_management_endpoints import ( KeyProjectBindingError, @@ -8911,7 +8915,10 @@ async def test_delete_cache_key_object_broadcasts_invalidation(monkeypatch): # Real, observable state: the cache object itself no longer holds the entry. assert real_user_api_key_cache.get_cache(key="hashed-broadcast-key") is None - mock_publish.assert_awaited_once_with(cache_key="hashed-broadcast-key") + assert mock_publish.await_args_list == [ + call(cache_key="hashed-broadcast-key"), + call(cache_key=budget_windows_absent_cache_key("key", "hashed-broadcast-key")), + ] @pytest.mark.asyncio diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index e33743bb149..2f4dad0ccd7 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -2684,7 +2684,7 @@ async def test_team_block_unblock_reach_warmed_auth_on_every_worker( from litellm.proxy._types import BlockTeamRequest from litellm.proxy.auth.auth_checks import get_team_object from litellm.proxy.auth.team_grants import team_model_aliases - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, budget_windows_absent_cache_key from litellm.proxy.management_endpoints import team_endpoints team_id: Final = "team-kill-switch" @@ -2721,7 +2721,11 @@ async def test_team_block_unblock_reach_warmed_auth_on_every_worker( seen_by_auth: Final = await get_team_object(team_id=team_id, prisma_client=prisma_client, user_api_key_cache=cache) assert seen_by_auth.blocked is blocked assert team_model_aliases(seen_by_auth) == {"fast": "gpt-4o"} - assert redis.broadcast_keys == [f"team_id:{team_id}", "team_alias:kill-switch"] + assert redis.broadcast_keys == [ + f"team_id:{team_id}", + "team_alias:kill-switch", + budget_windows_absent_cache_key("team", team_id), + ] assert (response.blocked, response.litellm_model_table, response.object_permission) == (blocked, None, None) diff --git a/tests/unit/proxy/test_budget_reservation.py b/tests/unit/proxy/test_budget_reservation.py index 535c3da8c3a..6ce8d400daf 100644 --- a/tests/unit/proxy/test_budget_reservation.py +++ b/tests/unit/proxy/test_budget_reservation.py @@ -66,7 +66,7 @@ def spend_counter_state(): original_prisma_client = ps.prisma_client counter_cache = DualCache() - key_cache = DualCache() + key_cache = UserApiKeyCache() ps.spend_counter_cache = counter_cache ps.user_api_key_cache = key_cache ps.prisma_client = None diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 76d94628d81..d09008d62bc 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import dataclasses import importlib import json import logging @@ -10,9 +11,10 @@ import subprocess import sys import time import types -from collections.abc import Callable +from collections.abc import Callable, Iterator, Mapping, Sequence from datetime import datetime, timedelta, timezone from pathlib import Path +from types import MappingProxyType from typing import Final from unittest import mock from unittest.mock import AsyncMock, MagicMock, call, create_autospec, mock_open, patch @@ -48,6 +50,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.login_throttle import LoginThrottle from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendUpdateQueue from litellm.proxy.hooks.parallel_request_limiter_v3 import RequestRateLimiterStash from litellm.proxy.proxy_server import app, aws_error_type_exception_handler, initialize, openai_exception_handler from litellm.utils import _invalidate_model_cost_lowercase_map @@ -14075,17 +14078,18 @@ def test_startup_is_silent_when_mock_testing_params_disabled(caplog): @contextlib.contextmanager -def _window_spend_enqueue_env(cached_objects: dict): +def _window_spend_enqueue_env( + cached_objects: Mapping[str, object], prisma_client: MagicMock | None = None +) -> Iterator[WindowSpendUpdateQueue]: """Point increment_spend_counters at throwaway caches and a real WindowSpendUpdateQueue, and hand back the queue to inspect.""" import litellm.proxy.proxy_server as ps from litellm.caching.dual_cache import DualCache - from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( - WindowSpendUpdateQueue, - ) + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache - user_api_key_cache = MagicMock() - user_api_key_cache.async_get_cache = AsyncMock(side_effect=lambda key, **_: cached_objects.get(key)) + user_api_key_cache: Final = UserApiKeyCache() + for key, value in cached_objects.items(): + user_api_key_cache.set_cache(key=key, value=value) queue = WindowSpendUpdateQueue() proxy_logging_obj = MagicMock() @@ -14099,7 +14103,7 @@ def _window_spend_enqueue_env(cached_objects: dict): ) ps.user_api_key_cache = user_api_key_cache ps.spend_counter_cache = DualCache() - ps.prisma_client = None + ps.prisma_client = prisma_client ps.proxy_logging_obj = proxy_logging_obj try: yield queue @@ -14287,6 +14291,293 @@ async def test_team_window_spend_row_carries_the_request_start_time(): assert enqueued[0]["started_at"] == "2026-08-10T12:00:00.500000" +_WINDOW_RESET_AT: Final = "2026-08-17T00:00:00+00:00" + + +def _window(budget_duration: str, max_budget: float) -> dict[str, object]: + return {"budget_duration": budget_duration, "max_budget": max_budget, "reset_at": _WINDOW_RESET_AT} + + +@dataclasses.dataclass(frozen=True, slots=True) +class _StoredRow: + budget_limits: list[dict[str, object]] | None + + +def _prisma_with_rows( + keys: Mapping[str, _StoredRow] = MappingProxyType({}), + teams: Mapping[str, _StoredRow] = MappingProxyType({}), +) -> MagicMock: + """Key and team tables that answer find_unique from the given rows.""" + client: Final = MagicMock() + client.db.litellm_verificationtoken.find_unique = AsyncMock(side_effect=lambda where: keys.get(where["token"])) + client.db.litellm_teamtable.find_unique = AsyncMock(side_effect=lambda where: teams.get(where["team_id"])) + return client + + +def _charged_windows(enqueued: Sequence[Mapping[str, object]]) -> list[tuple[object, object, object]]: + return [(item["entity_id"], item["window_duration"], item["spend"]) for item in enqueued] + + +@pytest.mark.asyncio +async def test_key_window_is_charged_when_the_cached_key_expired_before_the_request_ended(): + """A stream that outlives user_api_key_cache_ttl finds no cached key when its + cost lands. Its spend still belongs to the key's budget windows.""" + import litellm.proxy.proxy_server as ps + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows(keys={"hashed-token": _StoredRow([_window("30d", 100.0)])}) + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) + enqueued: Final = await _drain(queue) + assert await ps.user_api_key_cache.async_get_cache(key="hashed-token") is None + + assert [item["entity_type"] for item in enqueued] == ["key"] + assert _charged_windows(enqueued) == [("hashed-token", "30d", pytest.approx(0.25))] + + +@pytest.mark.asyncio +async def test_team_window_is_charged_when_the_cached_team_expired_before_the_request_ended(): + import litellm.proxy.proxy_server as ps + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows(teams={"team-1": _StoredRow([_window("7d", 50.0)])}) + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=1.5) + enqueued: Final = await _drain(queue) + assert await ps.user_api_key_cache.async_get_cache(key="team_id:team-1") is None + + assert [item["entity_type"] for item in enqueued] == ["team"] + assert _charged_windows(enqueued) == [("team-1", "7d", pytest.approx(1.5))] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "token, keys", + [ + pytest.param("litellm_proxy_master_key", {}, id="no-key-row"), + pytest.param("hashed-token", {"hashed-token": _StoredRow(budget_limits=None)}, id="key-row-without-windows"), + ], +) +async def test_a_token_without_budget_windows_reads_the_database_once_across_requests( + token: str, keys: Mapping[str, _StoredRow] +): + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows(keys=keys) + find_unique: Final = prisma_client.db.litellm_verificationtoken.find_unique + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + await increment_spend_counters(token=token, team_id=None, user_id=None, response_cost=0.25) + reads_after_first_request: Final = find_unique.await_count + for _ in range(3): + await increment_spend_counters(token=token, team_id=None, user_id=None, response_cost=0.25) + enqueued: Final = await _drain(queue) + + assert enqueued == [] + assert reads_after_first_request > 0 + assert find_unique.await_count == reads_after_first_request + + +@pytest.mark.asyncio +async def test_every_cache_miss_charges_the_windows_the_key_row_holds_at_that_moment(): + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows(keys={"hashed-token": _StoredRow([_window("30d", 100.0)])}) + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=_StoredRow([_window("7d", 5.0)]) + ) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.5) + enqueued: Final = await _drain(queue) + + assert sorted(_charged_windows(enqueued), key=lambda charge: str(charge[1])) == [ + ("hashed-token", "30d", pytest.approx(0.25)), + ("hashed-token", "7d", pytest.approx(0.5)), + ] + + +@pytest.mark.asyncio +async def test_a_failed_key_row_read_skips_that_request_and_the_next_one_reads_again(): + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows() + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + side_effect=ConnectionError("database is unreachable") + ) + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) + after_failure: Final = await _drain(queue) + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=_StoredRow([_window("7d", 5.0)]) + ) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.5) + after_recovery: Final = await _drain(queue) + + assert after_failure == [] + assert _charged_windows(after_recovery) == [("hashed-token", "7d", pytest.approx(0.5))] + + +@pytest.mark.asyncio +async def test_a_late_request_after_a_key_update_charges_windows_the_key_did_not_have_before(): + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.auth_checks import delete_cache_key_object + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows(keys={"hashed-token": _StoredRow(budget_limits=None)}) + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=_StoredRow([_window("7d", 5.0)]) + ) + await delete_cache_key_object( + hashed_token="hashed-token", user_api_key_cache=ps.user_api_key_cache, proxy_logging_obj=None + ) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.5) + enqueued: Final = await _drain(queue) + + assert _charged_windows(enqueued) == [("hashed-token", "7d", pytest.approx(0.5))] + + +@pytest.mark.asyncio +async def test_a_key_update_that_lands_during_the_row_read_does_not_hide_the_windows_it_adds(): + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.auth_checks import delete_cache_key_object + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows() + updated_row: Final = _StoredRow([_window("7d", 5.0)]) + + async def read_the_row_from_before_the_update(where: Mapping[str, str]) -> _StoredRow: + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=updated_row) + await delete_cache_key_object( + hashed_token=where["token"], user_api_key_cache=ps.user_api_key_cache, proxy_logging_obj=None + ) + return _StoredRow(budget_limits=None) + + prisma_client.db.litellm_verificationtoken.find_unique = read_the_row_from_before_the_update + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + ps.spend_counter_cache.set_cache(key="spend:key:hashed-token", value=0.0) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) + _ = await _drain(queue) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.5) + after_the_update: Final = await _drain(queue) + + assert _charged_windows(after_the_update) == [("hashed-token", "7d", pytest.approx(0.5))] + + +@pytest.mark.asyncio +async def test_late_requests_from_many_keys_with_windows_do_not_push_out_a_known_no_window_key(): + import litellm.proxy.proxy_server as ps + from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY + from litellm.proxy.proxy_server import increment_spend_counters + + windowed_tokens: Final = tuple(f"windowed-{index}" for index in range(DEFAULT_MAX_SIZE_IN_MEMORY)) + rows: Final = { + "no-windows": _StoredRow(budget_limits=None), + **{token: _StoredRow([_window("7d", 5.0)]) for token in windowed_tokens}, + } + prisma_client: Final = _prisma_with_rows(keys=rows) + find_unique: Final = prisma_client.db.litellm_verificationtoken.find_unique + + def no_window_key_reads() -> int: + return sum(1 for call in find_unique.await_args_list if call.kwargs["where"]["token"] == "no-windows") + + with _window_spend_enqueue_env({}, prisma_client=prisma_client): + await increment_spend_counters(token="no-windows", team_id=None, user_id=None, response_cost=0.1) + reads_after_the_first_request: Final = no_window_key_reads() + for token in windowed_tokens: + await increment_spend_counters(token=token, team_id=None, user_id=None, response_cost=0.1) + ps.spend_counter_cache.set_cache(key="spend:key:no-windows", value=0.1) + await increment_spend_counters(token="no-windows", team_id=None, user_id=None, response_cost=0.1) + + assert no_window_key_reads() == reads_after_the_first_request + + +@pytest.mark.asyncio +async def test_a_slow_window_row_read_does_not_hold_back_the_key_spend_counter(): + import litellm.proxy.proxy_server as ps + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows() + read_started: Final = asyncio.Event() + finish_read: Final = asyncio.Event() + + async def slow_row_read(where: Mapping[str, str]) -> _StoredRow: + read_started.set() + await finish_read.wait() + return _StoredRow([_window("7d", 5.0)]) + + prisma_client.db.litellm_verificationtoken.find_unique = slow_row_read + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + ps.spend_counter_cache.set_cache(key="spend:key:hashed-token", value=0.0) + ps.spend_counter_cache.set_cache(key="spend:key:hashed-token:window:7d", value=1.0) + charge: Final = asyncio.create_task( + increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) + ) + assert await asyncio.wait_for(read_started.wait(), timeout=5) + key_spend_while_the_row_read_waits: Final = ps.spend_counter_cache.get_cache(key="spend:key:hashed-token") + finish_read.set() + await charge + window_spend: Final = ps.spend_counter_cache.get_cache(key="spend:key:hashed-token:window:7d") + charged: Final = await _drain(queue) + + assert key_spend_while_the_row_read_waits == pytest.approx(0.25) + assert window_spend == pytest.approx(1.25) + assert _charged_windows(charged) == [("hashed-token", "7d", pytest.approx(0.25))] + + +@pytest.mark.asyncio +async def test_a_late_request_that_misses_the_updated_team_charges_windows_the_team_did_not_have_before(): + """Another worker's /team/update only evicts this worker's cached team, so its next late cost reads the row.""" + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team + from litellm.proxy.proxy_server import increment_spend_counters + + prisma_client: Final = _prisma_with_rows(teams={"team-1": _StoredRow(budget_limits=None)}) + updated_row: Final = _StoredRow([_window("7d", 50.0)]) + + with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue: + await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=1.5) + prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=updated_row) + await refresh_cached_team( + team_row=LiteLLM_TeamTable(team_id="team-1", budget_limits=updated_row.budget_limits), + user_api_key_cache=ps.user_api_key_cache, + proxy_logging_obj=ps.proxy_logging_obj, + ) + ps.user_api_key_cache.delete_cache(key="team_id:team-1") + await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=2.0) + enqueued: Final = await _drain(queue) + + assert _charged_windows(enqueued) == [("team-1", "7d", pytest.approx(2.0))] + + +@pytest.mark.asyncio +async def test_late_costs_for_many_tokens_without_windows_leave_the_cached_team_in_place(): + from litellm.proxy.proxy_server import increment_spend_counters + + team_obj: Final = MagicMock() + team_obj.budget_limits = [_window("7d", 50.0)] + + with _window_spend_enqueue_env({"team_id:team-1": team_obj}, prisma_client=_prisma_with_rows()) as queue: + for token_number in range(250): + await increment_spend_counters( + token=f"token-{token_number}", team_id=None, user_id=None, response_cost=0.01 + ) + await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=1.5) + enqueued: Final = await _drain(queue) + + assert _charged_windows(enqueued) == [("team-1", "7d", pytest.approx(1.5))] + + def _mock_startup_prisma_client(health_check_error=None, connect_error=None): client = MagicMock() client.connect = AsyncMock(side_effect=connect_error) From 7fe6b1de34a5374b7283e5af29ad3dcc68b13f20 Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:58:46 -0800 Subject: [PATCH 05/15] fix(proxy)!: ignore stored general_settings keys no settings API manages (#48967) Apply database general settings only for managed keys, warn about ignored copies, and allow admins to delete stale stored keys Move intended database-only settings into config.yaml before upgrading, including authentication and access restrictions --- .../proxy/config_resolvers/settings_store.py | 19 +- litellm/proxy/proxy_server.py | 108 ++++++--- .../configuration/test_effective_settings.py | 56 ++++- .../config_resolvers/test_settings_store.py | 31 +++ .../proxy/proxy_server/test_proxy_config.py | 134 ++++++++++-- ...test_proxy_server_endpoints_and_startup.py | 206 +++++++++++++++++- 6 files changed, 494 insertions(+), 60 deletions(-) diff --git a/litellm/proxy/config_resolvers/settings_store.py b/litellm/proxy/config_resolvers/settings_store.py index 70486be0068..3be66821915 100644 --- a/litellm/proxy/config_resolvers/settings_store.py +++ b/litellm/proxy/config_resolvers/settings_store.py @@ -42,10 +42,12 @@ _EMPTY_ROWS: Final[Mapping[DbRow, Mapping[str, JsonValue]]] = MappingProxyType({ class SettingsStore(MutableMapping[str, JsonValue]): - def __init__(self, section: Section) -> None: + def __init__(self, section: Section, *, managed_db_keys: frozenset[str] | None = None) -> None: self._section: Final = section + self._managed_db_keys: Final = managed_db_keys self._yaml_values: Mapping[str, JsonValue] = _EMPTY_VALUES self._database_rows: Mapping[DbRow, Mapping[str, JsonValue]] = _EMPTY_ROWS + self._unmanaged_db_keys: frozenset[str] = frozenset() self._runtime_values: Mapping[str, JsonValue] = _EMPTY_VALUES self._deleted_runtime_keys: frozenset[str] = frozenset() @@ -77,15 +79,24 @@ class SettingsStore(MutableMapping[str, JsonValue]): return self.owned_by_config(key) and self._db_value_is_shadowed(key) def apply_db_row(self, row: DbRow, db_row: Mapping[str, JsonValue]) -> None: + applied_row: Final = MappingProxyType({key: db_row[key] for key in db_row if self._db_manages(row, key)}) previous_row: Final = self._database_rows.get(row, _EMPTY_VALUES) changed: Final = frozenset( key - for key in (*previous_row, *db_row) - if previous_row.get(key, ABSENT) != db_row.get(key, ABSENT) # pyright: ignore[reportUnknownArgumentType] # JsonValue vs Absent compare + for key in (*previous_row, *applied_row) + if previous_row.get(key, ABSENT) != applied_row.get(key, ABSENT) # pyright: ignore[reportUnknownArgumentType] # JsonValue vs Absent compare ) - self._database_rows = MappingProxyType({**self._database_rows, row: MappingProxyType(dict(db_row))}) + self._database_rows = MappingProxyType({**self._database_rows, row: applied_row}) + if row == self._section: + self._unmanaged_db_keys = frozenset(key for key in db_row if key not in applied_row) self._clear_runtime_keys(changed) + def unmanaged_db_keys(self) -> tuple[str, ...]: + return tuple(sorted(self._unmanaged_db_keys)) + + def _db_manages(self, row: DbRow, key: str) -> bool: + return row != self._section or self._managed_db_keys is None or key in self._managed_db_keys + def resolved(self) -> Mapping[str, JsonValue]: return MappingProxyType(dict(self)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 22e5bd6b3bd..33dcf978854 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5722,6 +5722,14 @@ def _reload_settings_store(section: Section, store: SettingsStore, section_confi store["pass_through_endpoints"] = serving_pass_throughs +def _general_settings_row_is_readable(row: _ConfigParamValueRow | None) -> bool: + return row is None or row.param_value is None or isinstance(row.param_value, Mapping) + + +def _db_managed_general_settings_keys() -> frozenset[str]: + return frozenset((*ConfigGeneralSettings.model_fields, *_CLEANUP_SCHEDULE_KEYS, "allowed_ips", "email_settings")) + + def _bind_general_settings_store(settings: SettingsStore) -> None: global general_settings general_settings = settings # pyright: ignore[reportAssignmentType] # legacy global accepts mappings @@ -5872,7 +5880,9 @@ class ProxyConfig: # whether an existing request predates the prices it just fetched, and re-serving one # costs a single fetch where skipping one leaves it priced wrong indefinitely self.model_cost_map_applied_revision: int = 0 - self.settings: Final[SettingsStore] = SettingsStore("general_settings") + self.settings: Final[SettingsStore] = SettingsStore( + "general_settings", managed_db_keys=_db_managed_general_settings_keys() + ) self.router_settings: Final[SettingsStore] = SettingsStore("router_settings") self.litellm_settings: Final[SettingsStore] = SettingsStore("litellm_settings") self.environment_variables: Final[SettingsStore] = SettingsStore("environment_variables") @@ -5883,6 +5893,8 @@ class ProxyConfig: self._db_environment_keys: frozenset[str] = frozenset() self._sso_environment_removals: frozenset[str] = frozenset() self._warned_shadowed_keys: frozenset[tuple[Section, str]] = frozenset() + self._warned_unmanaged_keys: frozenset[str] = frozenset() + self._warned_unreadable_db_general_settings: bool = False self._db_config_callback_entries: Mapping[tuple[str, str], _DbCallbackRegistration] = MappingProxyType({}) self._settings_stores: Final[Mapping[Section, SettingsStore]] = MappingProxyType( { @@ -8406,15 +8418,20 @@ class ProxyConfig: general_settings.get("maximum_spend_logs_retention_interval", "1d"), ) - async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | None) -> None: + async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | str | None) -> None: global general_settings if db_general_settings is None: return + if not isinstance(db_general_settings, Mapping): + self._warn_about_unreadable_db_general_settings() + return + self._warned_unreadable_db_general_settings = False if not isinstance(general_settings, SettingsStore): self.settings.load_yaml(_as_settings_mapping(general_settings)) cache_size_was_db: Final = self.settings.source("user_api_key_cache_max_size") == "db" previous_cleanup_schedule: Final = self._resolved_cleanup_schedule() self.settings.apply_db_row("general_settings", db_general_settings) + self._warn_about_unmanaged_db_settings() _bind_general_settings_store(self.settings) await self._apply_general_settings_side_effects( db_general_settings, @@ -8632,6 +8649,7 @@ class ProxyConfig: ) self._warn_about_shadowed_db_settings() + self._warn_about_unmanaged_db_settings() return self._config_with_resolved_settings(config) async def _apply_db_environment_variables_row( @@ -8690,6 +8708,30 @@ class ProxyConfig: ) self._warned_shadowed_keys = shadowed + def _warn_about_unmanaged_db_settings(self) -> None: + unmanaged: Final = frozenset( + key for key in self.settings.unmanaged_db_keys() if key not in _GENERAL_SETTINGS_UI_LITELLM_FIELDS + ) + for key in sorted(unmanaged - self._warned_unmanaged_keys): + verbose_proxy_logger.warning( + "general_settings.%s is stored in the database, but the proxy does not apply that stored value, so it " + "is ignored. %s", + key, + "The config file's value applies. Remove the stored copy with POST /config/field/delete." + if self.settings.owned_by_config(key) + else "Set it in the config file to use it, or remove it with POST /config/field/delete.", + ) + self._warned_unmanaged_keys = unmanaged + + def _warn_about_unreadable_db_general_settings(self) -> None: + if self._warned_unreadable_db_general_settings: + return + verbose_proxy_logger.warning( + "general_settings is stored in the database as something other than a JSON object, so the proxy " + "ignores every stored general setting." + ) + self._warned_unreadable_db_general_settings = True + def _prepared_db_settings_values(self, section: Section, value: object) -> Mapping[str, SettingsJsonValue]: scrubbed: Final = _scrub_db_overlay_remote_module_loads(section=section, db_value=value) return _as_settings_mapping(scrubbed) @@ -8857,7 +8899,10 @@ class ProxyConfig: ) # initialize vector stores, guardrails, etc. table in db - await self._init_non_llm_objects_in_db(prisma_client=prisma_client) + await self._init_non_llm_objects_in_db( + prisma_client=prisma_client, + refresh_pass_through_endpoints=_general_settings_row_is_readable(db_general_settings), + ) except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e) @@ -8926,7 +8971,9 @@ class ProxyConfig: except Exception as e: # noqa: BLE001 # best-effort: a failing stop must not break proxy shutdown verbose_proxy_logger.error("Error stopping auth cache invalidation subscriber: %s", e) - async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): + async def _init_non_llm_objects_in_db( + self, prisma_client: PrismaClient, *, refresh_pass_through_endpoints: bool = True + ): """ Use this to read non-llm objects from the db and initialize them @@ -8950,7 +8997,7 @@ class ProxyConfig: if self._should_load_db_object(object_type="agents"): await self._init_agents_in_db(prisma_client=prisma_client) - if self._should_load_db_object(object_type="pass_through_endpoints"): + if refresh_pass_through_endpoints and self._should_load_db_object(object_type="pass_through_endpoints"): await self._init_pass_through_endpoints_in_db() if self._should_load_db_object(object_type="prompts"): @@ -21111,53 +21158,56 @@ async def delete_config_general_settings( if data.field_name in _GENERAL_SETTINGS_UI_LITELLM_FIELDS: return await _reset_general_settings_ui_litellm_field(data.field_name, user_api_key_dict) - if data.field_name not in ConfigGeneralSettings.model_fields: + config_table: Final[TableActions[_ConfigParamValueRow]] = _config_param_table(prisma_client) + stored_row: Final = await config_table.find_first(where={"param_name": "general_settings"}) + stored: Final = ( + _SETTINGS_MAPPING.validate_python(stored_row.param_value) + if stored_row is not None and isinstance(stored_row.param_value, Mapping) + else None + ) + + if not _general_settings_row_is_readable(stored_row): + raise HTTPException( + status_code=400, + detail={ + "error": "general_settings is stored in the database as something other than a JSON object, so no " + "field can be removed from it" + }, + ) + + if data.field_name not in ConfigGeneralSettings.model_fields and (stored is None or data.field_name not in stored): raise HTTPException( status_code=400, detail={"error": f"Invalid field={data.field_name} passed in."}, ) - proxy_config.reject_config_owned_deletes(section_name="general_settings", keys=(data.field_name,)) + if data.field_name in _db_managed_general_settings_keys(): + proxy_config.reject_config_owned_deletes(section_name="general_settings", keys=(data.field_name,)) - ## get general settings from db - db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first( - where={"param_name": "general_settings"} - ) - ### pop the value - - if db_general_settings is None or db_general_settings.param_value is None: + if stored is None: raise HTTPException( status_code=400, detail={"error": f"Field name={data.field_name} not in config"}, ) - else: - general_settings = dict(db_general_settings.param_value) - - before_general_settings: Final = copy.deepcopy(general_settings) - - ## update db - - general_settings.pop(data.field_name, None) + remaining: Final = {key: value for key, value in stored.items() if key != data.field_name} response: Final = await ConfigRepository(prisma_client).table.upsert( where={"param_name": "general_settings"}, data={ "create": { "param_name": "general_settings", - "param_value": json.dumps(general_settings), + "param_value": json.dumps(remaining), }, - "update": {"param_value": json.dumps(general_settings)}, + "update": {"param_value": json.dumps(remaining)}, }, ) await invalidate_config_param("general_settings") - proxy_config.settings.apply_db_row("general_settings", general_settings) + proxy_config.settings.apply_db_row("general_settings", remaining) if is_resource_list("general_settings", data.field_name): - stored_endpoints: Final = general_settings.get("pass_through_endpoints") + stored_endpoints: Final = remaining.get("pass_through_endpoints") await proxy_config.serve_pass_through_endpoints(stored_endpoints if isinstance(stored_endpoints, list) else ()) asyncio.create_task( - create_config_audit_log( - "general_settings", "deleted", before_general_settings, general_settings, user_api_key_dict - ) + create_config_audit_log("general_settings", "deleted", dict(stored), remaining, user_api_key_dict) ) return response diff --git a/tests/integration/configuration/test_effective_settings.py b/tests/integration/configuration/test_effective_settings.py index 07477f1f048..1b0086c619a 100644 --- a/tests/integration/configuration/test_effective_settings.py +++ b/tests/integration/configuration/test_effective_settings.py @@ -20,7 +20,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter from tests.integration._support.client import Gateway, eventually, object_value, string_value from tests.integration._support.database import read_rows, scratch_database -from tests.integration._support.process import owned_proxy +from tests.integration._support.process import owned_proxy, owned_proxy_process from tests.integration._support.wire import Reply, Request, Wire, wire_server pytestmark: Final = pytest.mark.shares_worker_state @@ -1701,3 +1701,57 @@ def test_db_environment_variables_saved_from_a_session_outside_the_tables_schema assert isinstance(first, int) and isinstance(second, int) assert 0 < first < second + + +def test_a_stored_general_setting_no_settings_api_manages_stays_off_and_the_rest_of_the_row_applies( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + alias: Final = "stale-database-setting-" + uuid.uuid4().hex + with scratch_database() as database_url, wire_server(lambda _request: _completion_reply("pong")) as provider: + monkeypatch.setenv("DATABASE_URL", database_url) + _push_schema() + with psycopg.connect(database_url, autocommit=True) as writer: + writer.execute( + 'INSERT INTO "LiteLLM_Config" (param_name, param_value) VALUES (%s, %s)', + ("general_settings", Jsonb({"enforce_user_param": True, "store_prompts_in_spend_logs": True})), + ) + config: Final = tmp_path / "stale-database-setting.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": alias, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": provider.url + "/v1", + "api_key": "integration-provider-key", + }, + } + ], + "general_settings": {"master_key": gateway.key, "store_model_in_db": True}, + } + ) + ) + with owned_proxy_process( + gateway, + tmp_path / "stale-database-setting", + {"DATABASE_URL": database_url, "DISABLE_SCHEMA_UPDATE": "true"}, + config=config, + database_setup=(), + ) as owned: + answer: Final = owned.gateway.request( + "POST", "/v1/chat/completions", {"model": alias, "messages": [{"role": "user", "content": "ping"}]} + ) + listed: Final = owned.gateway.request("GET", "/config/list", params={"config_type": "general_settings"}) + log: Final = owned.log.read_text() + + assert answer.status_code == 200, answer.text + assert listed.status_code == 200, listed.text + stored_prompts: Final = next( + entry + for entry in TypeAdapter(list[dict[str, JsonValue]]).validate_json(listed.content) + if entry["field_name"] == "store_prompts_in_spend_logs" + ) + assert (stored_prompts["field_value"], stored_prompts["source"]) == (True, "db") + assert "general_settings.enforce_user_param is stored in the database, but the proxy does not apply" in log diff --git a/tests/unit/proxy/config_resolvers/test_settings_store.py b/tests/unit/proxy/config_resolvers/test_settings_store.py index 7b2cd404b46..76850825937 100644 --- a/tests/unit/proxy/config_resolvers/test_settings_store.py +++ b/tests/unit/proxy/config_resolvers/test_settings_store.py @@ -471,3 +471,34 @@ def test_settings_store_truthiness_matches_emptiness() -> None: assert bool(store) is True del store["max_parallel_requests"] assert bool(store) is False + + +def test_settings_store_ignores_stored_keys_outside_its_managed_db_keys_and_still_applies_them_from_yaml() -> None: + store: Final = SettingsStore("general_settings", managed_db_keys=frozenset({"max_parallel_requests"})) + stored_row: Final[dict[str, JsonValue]] = {"enforce_user_param": True, "max_parallel_requests": 5} + + store.apply_db_row("general_settings", stored_row) + + assert dict(store) == {"max_parallel_requests": 5} + assert store.source("enforce_user_param") == "unset" + assert store.unmanaged_db_keys() == ("enforce_user_param",) + + store.load_yaml({"enforce_user_param": True}) + store.apply_db_row("general_settings", stored_row) + + assert dict(store) == {"enforce_user_param": True, "max_parallel_requests": 5} + assert store.source("enforce_user_param") == "config" + assert store.unmanaged_db_keys() == ("enforce_user_param",) + + +def test_settings_store_managed_db_keys_leave_other_rows_and_unrestricted_stores_whole() -> None: + restricted: Final = SettingsStore("general_settings", managed_db_keys=frozenset()) + unrestricted: Final = SettingsStore("general_settings") + + restricted.apply_db_row("ui_settings", {"disable_custom_api_keys": True}) + unrestricted.apply_db_row("general_settings", {"enforce_user_param": True}) + + assert dict(restricted) == {"disable_custom_api_keys": True} + assert restricted.unmanaged_db_keys() == () + assert dict(unrestricted) == {"enforce_user_param": True} + assert unrestricted.unmanaged_db_keys() == () diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index d9ff7fef9f9..a78209e16fe 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -102,7 +102,9 @@ def test_proxy_config_upserts_existing_deployment_without_duplicating_it(monkeyp @pytest.mark.asyncio -async def test_proxy_config_preserves_models_when_config_read_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_proxy_config_preserves_models_when_config_read_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: from litellm.proxy import proxy_server router = litellm.Router(model_list=[_deployment("config-deployment").to_json(exclude_none=True)]) @@ -135,6 +137,8 @@ async def test_proxy_config_keeps_database_models_and_deletes_stale_models( assert desired == frozenset({"kept-deployment"}) assert router.get_model_ids() == ["kept-deployment"] + + from tests._master_key import MASTER_KEY @@ -150,7 +154,6 @@ async def test_proxy_config_loads_lens_store_from_yaml(tmp_path, monkeypatch) -> assert is_lens_tracing_enabled(settings["tracing"], {}) is True - @pytest.mark.asyncio @pytest.mark.parametrize("shutdown_error", [False, True]) async def test_tracing_config_automatically_exports_spend_without_a_storage_dependency( @@ -176,7 +179,9 @@ async def test_tracing_config_automatically_exports_spend_without_a_storage_depe outcome: Final = pytest.raises(RuntimeError, match="shutdown failure") if shutdown_error else nullcontext() with outcome: async with manage_tracing(enabled=True, client_factory=client): - logger: Final = next(callback for callback in litellm._async_success_callback if isinstance(callback, LensExporter)) + logger: Final = next( + callback for callback in litellm._async_success_callback if isinstance(callback, LensExporter) + ) await logger.async_log_success_event( {"standard_logging_object": {"id": "response-1", "response_cost": 0.25}}, None, None, None ) @@ -2693,7 +2698,6 @@ async def test_load_config_rejects_non_boolean_deny_by_default( await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) - @pytest.mark.asyncio @pytest.mark.parametrize( "yaml_value", ["files-api-2025-04-14", "[files-api-2025-04-14, 7]"], ids=["bare-string", "non-string-entry"] @@ -2713,6 +2717,7 @@ async def test_load_config_rejects_a_malformed_vertex_drop_anthropic_betas_list( with pytest.raises(ValidationError, match="vertex_ai_pass_through_drop_anthropic_betas"): await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + @pytest.mark.asyncio async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path, monkeypatch): """Regression: router_settings.plugins dotted-path strings must be resolved to @@ -4470,6 +4475,86 @@ async def test_ProxyConfig__update_general_settings_updates_health_check_retenti reschedule.assert_awaited_once() +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_ignores_a_stored_key_no_settings_api_manages_and_warns_once( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + pc: Final = ProxyConfig() + stored_row: Final = {"enforce_user_param": True, "store_prompts_in_spend_logs": True} + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await pc._update_general_settings(stored_row) + await pc._update_general_settings(stored_row) + from litellm.proxy import proxy_server + + assert proxy_server.general_settings["store_prompts_in_spend_logs"] is True + assert "enforce_user_param" not in proxy_server.general_settings + warnings: Final = [record.getMessage() for record in caplog.records if "enforce_user_param" in record.getMessage()] + assert warnings == [ + "general_settings.enforce_user_param is stored in the database, but the proxy does not apply that stored " + "value, so it is ignored. Set it in the config file to use it, or remove it with POST /config/field/delete." + ] + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_warns_about_an_ignored_stored_key_the_config_file_also_sets( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + pc: Final = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {"enforce_user_param": False}}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await pc._update_general_settings({"enforce_user_param": True}) + from litellm.proxy import proxy_server + + assert proxy_server.general_settings["enforce_user_param"] is False + warnings: Final = [record.getMessage() for record in caplog.records if "enforce_user_param" in record.getMessage()] + assert warnings == [ + "general_settings.enforce_user_param is stored in the database, but the proxy does not apply that stored " + "value, so it is ignored. The config file's value applies. Remove the stored copy with POST " + "/config/field/delete." + ] + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_does_not_warn_about_a_stored_copy_of_an_admin_ui_field( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + pc: Final = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {}}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await pc._update_general_settings({"budget_rollover": True, "enforce_user_param": True}) + + warned: Final = [record.getMessage().split(" ")[0] for record in caplog.records if "is ignored" in record.getMessage()] + assert warned == ["general_settings.enforce_user_param"] + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_warns_once_each_time_the_stored_row_stops_being_an_object( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + pc: Final = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {}}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", pc.settings) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await pc._update_general_settings(None) + await pc._update_general_settings('{"store_prompts_in_spend_logs": true}') + await pc._update_general_settings('{"store_prompts_in_spend_logs": true}') + await pc._update_general_settings({"store_prompts_in_spend_logs": True}) + await pc._update_general_settings('{"store_prompts_in_spend_logs": true}') + + warnings: Final = [record.getMessage() for record in caplog.records if "JSON object" in record.getMessage()] + assert warnings == [ + "general_settings is stored in the database as something other than a JSON object, so the proxy ignores " + "every stored general setting." + ] * 2 + + def _paused_scheduler(monkeypatch): from apscheduler.schedulers.asyncio import AsyncIOScheduler @@ -4918,9 +5003,6 @@ async def test_ProxyConfig__update_general_settings_none_input_noop(): # None input returns early. result = await pc._update_general_settings(db_general_settings=None) assert result is None - # Error-style: dict() will fail on non-mapping non-None input. - with pytest.raises(TypeError): - await pc._update_general_settings(db_general_settings=12345) # type: ignore[arg-type] @pytest.mark.asyncio @@ -5080,7 +5162,9 @@ async def test_ProxyConfig__update_config_from_db_keeps_keys_the_config_file_omi @pytest.mark.asyncio @pytest.mark.parametrize("ui_settings_already_synced", [False, True]) -async def test_ProxyConfig_add_deployment_continues_after_null_pass_through_endpoints(monkeypatch, ui_settings_already_synced): +async def test_ProxyConfig_add_deployment_continues_after_null_pass_through_endpoints( + monkeypatch: pytest.MonkeyPatch, ui_settings_already_synced: bool +) -> None: from litellm.proxy import proxy_server pc = ProxyConfig() @@ -5106,6 +5190,32 @@ async def test_ProxyConfig_add_deployment_continues_after_null_pass_through_endp assert settings_refresh.await_count == (0 if ui_settings_already_synced else 1) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stored_row, rebuilds_routes", + [({"pass_through_endpoints": []}, True), ('{"pass_through_endpoints": []}', False), ([], False)], + ids=["object", "json-string", "array"], +) +async def test_ProxyConfig_add_deployment_rebuilds_pass_through_routes_only_from_an_object_general_settings_row( + monkeypatch: pytest.MonkeyPatch, stored_row: object, rebuilds_routes: bool +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.pass_through_endpoints import pass_through_endpoints + + pc: Final = ProxyConfig() + rebuild_routes: Final = AsyncMock() + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prefetch_config_params", AsyncMock()) + monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=SimpleNamespace(param_value=stored_row))) + monkeypatch.setattr(pass_through_endpoints, "initialize_pass_through_endpoints_in_db", rebuild_routes) + monkeypatch.setattr(pc, "_should_load_db_object", lambda *, object_type: object_type == "pass_through_endpoints") + monkeypatch.setattr(pc, "get_credentials", AsyncMock()) + + await pc.add_deployment(prisma_client=MagicMock(), proxy_logging_obj=MagicMock(), ui_settings_already_synced=True) + + assert rebuild_routes.await_count == (1 if rebuilds_routes else 0) + + # --------------------------------------------------------------------------- # ProxyConfig._update_config_from_db # --------------------------------------------------------------------------- @@ -5285,9 +5395,7 @@ def revision_test_environment(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv(key) -def _environment_variables_row( - values: Mapping[str, str | None] | None, revision: int | None -) -> SimpleNamespace | None: +def _environment_variables_row(values: Mapping[str, str | None] | None, revision: int | None) -> SimpleNamespace | None: if values is None: return None return SimpleNamespace( @@ -5473,9 +5581,7 @@ async def test_environment_variables_row_read_before_an_overlapping_newer_sync_i await newer_sync_finished.wait() older_sync: Final = asyncio.create_task( - _sync_from( - pc, _config_database(read_older_row, AsyncMock(return_value=older_row), wait_for_the_newer_sync) - ) + _sync_from(pc, _config_database(read_older_row, AsyncMock(return_value=older_row), wait_for_the_newer_sync)) ) await older_environment_row_read.wait() await _sync_from(pc, _rows_database(newer_row, newer_row)) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index d09008d62bc..487638d6919 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1807,9 +1807,7 @@ class _DatabaseWithOneStoredCredential: @pytest.mark.asyncio -@pytest.mark.parametrize( - "encrypted_with, asks_to_migrate", [(PUBLICLY_KNOWN_KEY, True), ("sk-some-other-key", False)] -) +@pytest.mark.parametrize("encrypted_with, asks_to_migrate", [(PUBLICLY_KNOWN_KEY, True), ("sk-some-other-key", False)]) async def test_proxy_startup_asks_to_migrate_only_when_the_database_holds_values_under_the_unsafe_key( monkeypatch, tmp_path, encrypted_with, asks_to_migrate ): @@ -1927,9 +1925,7 @@ async def test_proxy_startup_names_the_config_file_that_set_the_unsafe_key(monke from litellm.proxy.auth.master_key_boot_check import UnsafeMasterKeyError from litellm.proxy.proxy_server import proxy_startup_event - config_path, announced = _boot_with_general_settings( - monkeypatch, tmp_path, {"master_key": PUBLICLY_KNOWN_KEY} - ) + config_path, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": PUBLICLY_KNOWN_KEY}) with pytest.raises(UnsafeMasterKeyError): async with proxy_startup_event(FastAPI()): @@ -3810,6 +3806,24 @@ async def test_db_reload_keeps_the_resolved_value_of_a_config_owned_env_referenc assert proxy_server_module.store_model_in_db is True +@pytest.mark.asyncio +async def test_db_reload_of_a_json_string_general_settings_row_reports_no_ignored_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server as proxy_server_module + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr(proxy_server_module, "user_api_key_cache", MagicMock(), raising=False) + proxy_config: Final = ProxyConfig() + proxy_config.settings.load_yaml({}) + monkeypatch.setattr(proxy_server_module, "general_settings", proxy_config.settings, raising=False) + + await proxy_config._update_general_settings('{"budget_rollover": true, "master_key": "sk-stored"}') + + assert proxy_config.settings.unmanaged_db_keys() == () + assert proxy_config.settings.source("master_key") != "db" + + def test_max_ui_session_budget_default_is_one_dollar(): """LIT-4662: the dashboard session budget default is a product decision; the old 0.25 default locked admins out of auto router Test Connection and the @@ -7298,7 +7312,7 @@ def test_deep_merge_dicts_skips_none_and_empty_lists(monkeypatch): db_param_value = { "max_parallel_requests": None, "allowed_models": [], - "new_key": "new_value", + "store_prompts_in_spend_logs": True, "nested": { "key1": "updated_value1", "key3": "value3", @@ -7311,7 +7325,7 @@ def test_deep_merge_dicts_skips_none_and_empty_lists(monkeypatch): assert result["max_parallel_requests"] == 10 assert result["allowed_models"] == ["gpt-3.5-turbo", "gpt-4"] - assert result["new_key"] == "new_value" + assert result["store_prompts_in_spend_logs"] is True assert result["nested"] == {"key1": "value1", "key2": "value2"} @@ -8397,10 +8411,18 @@ async def test_update_general_settings_db_pass_through_endpoint_overrides_yaml_e request.headers = {} request.query_params = {} - settings: Final = patch("litellm.proxy.proxy_server.general_settings", {"pass_through_endpoints": [yaml_endpoint]}) # test-quality-ok: the method reads this module global; no injection seam - yaml_endpoints: Final = patch("litellm.proxy.proxy_server.config_passthrough_endpoints", [yaml_endpoint]) # test-quality-ok: module global holding the YAML endpoints the fix merges in - initialize: Final = patch("litellm.proxy.proxy_server.initialize_pass_through_endpoints", AsyncMock()) # test-quality-ok: route registration needs the FastAPI app; auth is the observable here - master_key: Final = patch("litellm.proxy.proxy_server.master_key", MASTER_KEY) # test-quality-ok: a set master key is what makes a missing Authorization header a 401 + settings: Final = patch( + "litellm.proxy.proxy_server.general_settings", {"pass_through_endpoints": [yaml_endpoint]} + ) # test-quality-ok: the method reads this module global; no injection seam + yaml_endpoints: Final = patch( + "litellm.proxy.proxy_server.config_passthrough_endpoints", [yaml_endpoint] + ) # test-quality-ok: module global holding the YAML endpoints the fix merges in + initialize: Final = patch( + "litellm.proxy.proxy_server.initialize_pass_through_endpoints", AsyncMock() + ) # test-quality-ok: route registration needs the FastAPI app; auth is the observable here + master_key: Final = patch( + "litellm.proxy.proxy_server.master_key", MASTER_KEY + ) # test-quality-ok: a set master key is what makes a missing Authorization header a 401 with settings, yaml_endpoints, initialize, master_key: await ProxyConfig()._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]}) @@ -16135,6 +16157,166 @@ async def test_delete_config_general_settings_still_removes_a_key_the_database_o assert "max_request_size_mb" not in pc.settings +@pytest.mark.asyncio +async def test_delete_config_general_settings_removes_a_stored_key_no_settings_api_manages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._types import ConfigFieldDelete + from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings + + pc: Final = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {}}) + stored_row: Final = {"enforce_user_param": True, "store_prompts_in_spend_logs": True} + pc.settings.apply_db_row("general_settings", stored_row) + fake_prisma: Final = _fake_prisma_with_config(stored_row) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + + admin: Final = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + await delete_config_general_settings( + data=ConfigFieldDelete(field_name="enforce_user_param", config_type="general_settings"), + user_api_key_dict=admin, + ) + + written: Final = json.loads(fake_prisma.db.litellm_config.upsert.await_args.kwargs["data"]["update"]["param_value"]) + assert written == {"store_prompts_in_spend_logs": True} + assert pc.settings.unmanaged_db_keys() == () + + +@pytest.mark.asyncio +async def test_delete_config_general_settings_stops_serving_the_removed_pass_through_routes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from fastapi import FastAPI + from starlette.routing import Route + + from litellm.proxy._types import ConfigFieldDelete + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import _registered_pass_through_routes + from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings + from litellm.types.passthrough_endpoints.pass_through_endpoints import LITELLM_PASS_THROUGH_ENDPOINT_MARKER + + endpoint: Final = {"id": "stored-route", "path": "/stored-route", "target": "http://example.com"} + stored_row: Final = {"pass_through_endpoints": [endpoint], "store_prompts_in_spend_logs": True} + app: Final = FastAPI() + monkeypatch.setattr(proxy_server_module, "app", app) + monkeypatch.setattr(proxy_server_module, "premium_user", True) + monkeypatch.setattr(proxy_server_module, "config_passthrough_endpoints", None) + pc: Final = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {}}) + pc.settings.apply_db_row("general_settings", stored_row) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + monkeypatch.setattr(proxy_server_module, "prisma_client", _fake_prisma_with_config(stored_row)) + + def served_paths() -> frozenset[str]: + return frozenset( + route.path + for route in app.router.routes + if isinstance(route, Route) and getattr(route.endpoint, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, False) is True + ) + + saved_registry: Final = dict(_registered_pass_through_routes) + try: + _registered_pass_through_routes.clear() + await pc.serve_pass_through_endpoints([endpoint]) + served_before: Final = served_paths() + await delete_config_general_settings( + data=ConfigFieldDelete(field_name="pass_through_endpoints", config_type="general_settings"), + user_api_key_dict=UserAPIKeyAuth( + api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN + ), + ) + served_after: Final = served_paths() + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + assert "/stored-route" in served_before + assert "/stored-route" not in served_after + + +@pytest.mark.asyncio +async def test_delete_config_general_settings_removes_an_ignored_stored_key_the_config_file_also_sets( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._types import ConfigFieldDelete + from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings + + pc: Final = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {"enforce_user_param": False}}) + stored_row: Final = {"enforce_user_param": True, "store_prompts_in_spend_logs": True} + pc.settings.apply_db_row("general_settings", stored_row) + fake_prisma: Final = _fake_prisma_with_config(stored_row) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + + admin: Final = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + await delete_config_general_settings( + data=ConfigFieldDelete(field_name="enforce_user_param", config_type="general_settings"), + user_api_key_dict=admin, + ) + + written: Final = json.loads(fake_prisma.db.litellm_config.upsert.await_args.kwargs["data"]["update"]["param_value"]) + assert written == {"store_prompts_in_spend_logs": True} + assert pc.settings["enforce_user_param"] is False + assert pc.settings.unmanaged_db_keys() == () + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_row", ['{"max_request_size_mb": 42, "disable_custom_api_keys": true}', "not json"]) +@pytest.mark.parametrize("field_name", ["max_request_size_mb", "disable_custom_api_keys", "not_a_setting"]) +async def test_delete_config_general_settings_leaves_a_string_row_untouched( + monkeypatch: pytest.MonkeyPatch, stored_row: str, field_name: str +) -> None: + from litellm.proxy._types import ConfigFieldDelete + from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings + + pc: Final = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {}}) + fake_prisma: Final = _fake_prisma_with_config(stored_row) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + + admin: Final = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as refused: + await delete_config_general_settings( + data=ConfigFieldDelete(field_name=field_name, config_type="general_settings"), + user_api_key_dict=admin, + ) + + assert (refused.value.status_code, refused.value.detail) == ( + 400, + { + "error": "general_settings is stored in the database as something other than a JSON object, so no field " + "can be removed from it" + }, + ) + fake_prisma.db.litellm_config.upsert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_delete_config_general_settings_refuses_a_key_neither_managed_nor_stored( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._types import ConfigFieldDelete + from litellm.proxy.proxy_server import ProxyConfig, delete_config_general_settings + + pc: Final = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {}}) + fake_prisma: Final = _fake_prisma_with_config({"store_prompts_in_spend_logs": True}) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + + admin: Final = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as refused: + await delete_config_general_settings( + data=ConfigFieldDelete(field_name="enforce_user_param", config_type="general_settings"), + user_api_key_dict=admin, + ) + + assert refused.value.status_code == 400 + fake_prisma.db.litellm_config.upsert.assert_not_awaited() + + @pytest.mark.asyncio async def test_config_field_info_reports_the_declared_value_of_a_config_owned_secret(monkeypatch): from litellm.proxy.proxy_server import ProxyConfig, get_config_general_settings From a2d27881e87145058d652a481db1862acc4d5b94 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:08:10 -0800 Subject: [PATCH 06/15] fix(bedrock): reserve DeepSeek V3.2 output capacity and date GovCloud West Llama 3 (#49732) * fix(bedrock): set the regional DeepSeek V3.2 context to the model card 164K and date GovCloud West Llama 3 * fix(bedrock): reserve DeepSeek V3.2 output capacity and date GovCloud West Llama 3 --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 38 ++++++++++--------- model_prices_and_context_window.json | 38 ++++++++++--------- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 517aead5d29..7a430148a53 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13835,7 +13835,7 @@ "input_cost_per_token_flex": 3.7e-07, "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, @@ -13849,7 +13849,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14006,7 +14006,7 @@ "input_cost_per_token_flex": 3.7e-07, "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, @@ -14020,7 +14020,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14144,12 +14144,12 @@ "input_cost_per_token": 7.4e-07, "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md", "supports_audio_input": false, "supports_function_calling": true, "supports_native_structured_output": true, @@ -14254,7 +14254,7 @@ "input_cost_per_token_flex": 3.7e-07, "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, @@ -14268,7 +14268,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14790,7 +14790,7 @@ "input_cost_per_token_flex": 3.7e-07, "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, @@ -14804,7 +14804,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -15039,7 +15039,7 @@ "input_cost_per_token_flex": 3.1e-07, "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, @@ -15053,7 +15053,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -15160,7 +15160,7 @@ "input_cost_per_token_flex": 3.1e-07, "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, @@ -15174,7 +15174,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -15556,6 +15556,7 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md" }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -15566,6 +15567,7 @@ "supports_pdf_input": true }, "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -15712,7 +15714,7 @@ "input_cost_per_token_flex": 3.1e-07, "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, @@ -15726,7 +15728,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -24538,7 +24540,7 @@ "input_cost_per_token_flex": 3.1e-07, "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock_converse", - "max_input_tokens": 164000, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, @@ -24549,7 +24551,7 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 517aead5d29..7a430148a53 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13835,7 +13835,7 @@ "input_cost_per_token_flex": 3.7e-07, "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, @@ -13849,7 +13849,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14006,7 +14006,7 @@ "input_cost_per_token_flex": 3.7e-07, "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, @@ -14020,7 +14020,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14144,12 +14144,12 @@ "input_cost_per_token": 7.4e-07, "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md", "supports_audio_input": false, "supports_function_calling": true, "supports_native_structured_output": true, @@ -14254,7 +14254,7 @@ "input_cost_per_token_flex": 3.7e-07, "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, @@ -14268,7 +14268,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14790,7 +14790,7 @@ "input_cost_per_token_flex": 3.7e-07, "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, @@ -14804,7 +14804,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -15039,7 +15039,7 @@ "input_cost_per_token_flex": 3.1e-07, "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, @@ -15053,7 +15053,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -15160,7 +15160,7 @@ "input_cost_per_token_flex": 3.1e-07, "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, @@ -15174,7 +15174,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -15556,6 +15556,7 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md" }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -15566,6 +15567,7 @@ "supports_pdf_input": true }, "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 8000, @@ -15712,7 +15714,7 @@ "input_cost_per_token_flex": 3.1e-07, "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", - "max_input_tokens": 163840, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, @@ -15726,7 +15728,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md" }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -24538,7 +24540,7 @@ "input_cost_per_token_flex": 3.1e-07, "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock_converse", - "max_input_tokens": 164000, + "max_input_tokens": 156000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, @@ -24549,7 +24551,7 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false From 4e268c2c552eb26bbbfe37fd2c11342c94148073 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:40:32 +0000 Subject: [PATCH 07/15] test(gemini): assert vertex lists on content chunks (#49734) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/providers/test_gemini_long_decode_wire.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/providers/test_gemini_long_decode_wire.py b/tests/integration/providers/test_gemini_long_decode_wire.py index 5ecb5e2a942..90e7616e26c 100644 --- a/tests/integration/providers/test_gemini_long_decode_wire.py +++ b/tests/integration/providers/test_gemini_long_decode_wire.py @@ -380,7 +380,9 @@ def test_cut_chat_stream_hands_its_token_on_a_chunk_with_empty_vertex_ai_lists_a chunks: Final = payloads(post(gateway, "chat", request_body("chat", model, True)).text, True) carriers: Final = [chunk for chunk in chunks if "continuation_token" in json.dumps(chunk)] assert [_vertex_lists(chunk) for chunk in carriers] == [dict(_EMPTY_VERTEX_LISTS)], chunks - assert [_vertex_lists(chunk) for chunk in chunks if chunk not in carriers] == [{}] * (len(chunks) - 1), chunks + assert [_vertex_lists(chunk) for chunk in chunks] == [ + dict(_EMPTY_VERTEX_LISTS) if at(chunk, "choices", 0, "delta", "content") else {} for chunk in chunks + ], chunks metered: Final = [chunk for chunk in chunks if isinstance(chunk.get("usage"), dict)] assert [at(chunk, "usage", "completion_tokens_details") for chunk in metered] == [ {"text_tokens": CUT.output_tokens} From 01d6812f30c2ccee636b82aa27f8e8307400a222 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:41:16 -0800 Subject: [PATCH 08/15] fix(model-catalog): declare the thinking_on_by_default key (#49720) * fix(model-catalog): declare the thinking_on_by_default key The cost map guard runs main's catalog on a PR's cost map, so the key has to be declared on main before a PR can set it on the Claude 5 rows * chore(model-catalog): keep the attribute list sorted and document thinking_on_by_default --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm-rust/crates/model-catalog/src/wire/attributes.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 6ac7b4bb304..307f25f0bda 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -104,6 +104,8 @@ attributes! { /// Cloud regions the model is available in ('global' or region ids). supported_regions: Vec, thinking_always_on: bool, + /// Whether the model thinks when the request carries no thinking param. + thinking_on_by_default: bool, token_count_framing: u64 as NumericCount, token_count_model: TokenCountModel, token_count_responses_framing: u64 as NumericCount, @@ -301,6 +303,8 @@ mod tests { #[case::known_mode("mode", json!("chat"), true)] #[case::unknown_mode("mode", json!("chatt"), false)] #[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)] + #[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)] + #[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)] #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)] #[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)] #[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)] From 9601740d0590f945b9c225ea42380da0aa082deb Mon Sep 17 00:00:00 2001 From: shrey Date: Mon, 2 Nov 2026 15:53:39 -0800 Subject: [PATCH 09/15] fix(ui): make the off state of switches visible in light theme (#49718) The unchecked track used --input, which is nearly white in the light theme, so an off switch read as a white knob on a white page. The light unchecked track now uses --muted-foreground, and dark mode keeps its own override --- ui/litellm-dashboard/src/components/ui/switch.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/ui/switch.tsx b/ui/litellm-dashboard/src/components/ui/switch.tsx index 5d7171965d4..31233e049c5 100644 --- a/ui/litellm-dashboard/src/components/ui/switch.tsx +++ b/ui/litellm-dashboard/src/components/ui/switch.tsx @@ -16,7 +16,7 @@ function Switch({ data-slot="switch" data-size={size} className={cn( - "peer group/switch relative inline-flex shrink-0 items-center rounded-full border border-transparent shadow-xs transition-all outline-none after:absolute after:-inset-x-3 after:-inset-y-2 focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 data-[size=default]:h-[18.4px] data-[size=default]:w-[32px] data-[size=sm]:h-[14px] data-[size=sm]:w-[24px] dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 data-checked:bg-primary data-unchecked:bg-input dark:data-unchecked:bg-input/80 data-disabled:cursor-not-allowed data-disabled:opacity-50", + "peer group/switch relative inline-flex shrink-0 items-center rounded-full border border-transparent shadow-xs transition-all outline-none after:absolute after:-inset-x-3 after:-inset-y-2 focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 data-[size=default]:h-[18.4px] data-[size=default]:w-[32px] data-[size=sm]:h-[14px] data-[size=sm]:w-[24px] dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 data-checked:bg-primary data-unchecked:bg-muted-foreground dark:data-unchecked:bg-input/80 data-disabled:cursor-not-allowed data-disabled:opacity-50", className, )} {...props} From a5fe912ecf63ab73efddb0d59dca451d81839a57 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:55:10 +0000 Subject: [PATCH 10/15] fix(proxy): preserve DB settings during config reload (#49735) * fix(proxy): preserve DB settings during config reload Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): mark reload settings variable as final --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 17 ++++++++++++++++- .../proxy/proxy_server/test_proxy_config.py | 11 +++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 33dcf978854..91bbc418ffd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5717,7 +5717,6 @@ def _with_config_file_pass_through_endpoints( def _reload_settings_store(section: Section, store: SettingsStore, section_config: object) -> None: serving_pass_throughs: Final = store.get("pass_through_endpoints") store.load_yaml(_as_settings_mapping(section_config)) - store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING) if is_resource_list(section, "pass_through_endpoints") and serving_pass_throughs is not None: store["pass_through_endpoints"] = serving_pass_throughs @@ -6540,6 +6539,9 @@ class ProxyConfig: prisma_client=prisma_client, store_model_in_db=store_model_in_db, ) + else: + for section, store in self._settings_stores.items(): + store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING) config = self._config_with_resolved_settings(config) @@ -8623,12 +8625,24 @@ class ProxyConfig: environment_read: Final = next(self._environment_read_numbers) sections: Final = tuple(self._settings_stores) responses: Final = await asyncio.gather(*(get_config_param(prisma_client, section) for section in sections)) + general_settings_response: Final = responses[sections.index("general_settings")] + general_settings_value: Final[Mapping[str, SettingsJsonValue]] = ( + self._prepared_db_settings_values("general_settings", general_settings_response.param_value) + if general_settings_response is not None and general_settings_response.param_value is not None + else _EMPTY_SETTINGS_MAPPING + ) + self._settings_stores["general_settings"].apply_db_row("general_settings", general_settings_value) + response: _ConfigParamValueRow | None for section, response in zip(sections, responses): + if section == "general_settings": + continue if response is None: + self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING) continue param_value: JsonValue = getattr(response, "param_value", None) if param_value is None: + self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING) continue verbose_proxy_logger.debug( @@ -8639,6 +8653,7 @@ class ProxyConfig: if section == "litellm_settings": self._apply_litellm_settings_db_values(self._prepared_db_settings_values(section, param_value)) elif section == "environment_variables": + self._settings_stores[section].apply_db_row(section, _EMPTY_SETTINGS_MAPPING) await self._apply_db_environment_variables_row( prisma_client, response, environment_before_db, environment_read ) diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index a78209e16fe..55226fc629b 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -32,11 +32,13 @@ import litellm from litellm._logging import turn_on_json from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.config_resolvers.settings_store import SettingsStore from litellm.router_api import ApplyResult, RouterSettings from litellm.types.router import Deployment, LiteLLM_Params from litellm.proxy.proxy_server import ( ProxyConfig, _is_remote_module_url, + _reload_settings_store, _scrub_db_overlay_remote_module_loads, _scrub_guardrail_inner, resolve_complexity_router_plugins, @@ -70,6 +72,15 @@ def _db_model(model_id: str, model_name: str = "gpt-3.5-turbo") -> SimpleNamespa ) +def test_yaml_settings_reload_keeps_db_general_settings_until_sync() -> None: + settings: Final = SettingsStore("general_settings") + settings.apply_db_row("general_settings", {"max_file_downloads_per_minute": 2}) + + _reload_settings_store("general_settings", settings, {}) + + assert settings["max_file_downloads_per_minute"] == 2 + + @pytest.mark.asyncio async def test_proxy_config_adds_and_deletes_stale_deployment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy import proxy_server From 1bbfb3a02f8375e9ccdf5285497e30ea16491f31 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 2 Nov 2026 15:56:37 -0800 Subject: [PATCH 11/15] refactor(ui): move MCP text onto i18n (#49473) Route the page-authored English on /mcp-servers, /mcp/oauth/callback and the shared MCP permission, server selector and BYOK components through t() and Trans, with 1,094 messages added to en.json under mcpServers, mcpServerManagement, mcpTools and mcpOAuthCallback. The rendered English is unchanged. --- .../_components/AwsSigV4Fields.tsx | 273 +-- .../_components/CreateMCPServer.tsx | 134 +- .../_components/DcrBridgeToggle.tsx | 6 +- .../_components/EnvVarsSection.tsx | 73 +- .../_components/IdJagFormFields.tsx | 88 +- .../_components/ImportMCPServers.tsx | 25 +- .../_components/MCPGatewaySessionsTab.tsx | 146 +- .../_components/MCPLogoSelector.tsx | 19 +- .../_components/MCPNetworkSettings.tsx | 115 +- .../_components/MCPPermissionManagement.tsx | 118 +- .../mcp-servers/_components/MCPServerCard.tsx | 103 +- .../MCPServerUserCredentialsPanel.tsx | 77 +- .../_components/MCPStandardsSettings.tsx | 57 +- .../_components/MCPSubmissionsTab.tsx | 153 +- .../_components/MCPToolsetTableColumns.tsx | 55 +- .../_components/MCPToolsetsTab.tsx | 160 +- .../_components/OAuthFormFields.tsx | 204 +- .../_components/OpenAPIFormSection.tsx | 8 +- .../_components/OpenAPIQuickPicker.tsx | 10 +- .../_components/OpenApiByokFields.tsx | 23 +- .../PassthroughAuthorizeSection.tsx | 84 +- .../_components/StdioAvailability.tsx | 58 +- .../_components/StdioConfiguration.tsx | 10 +- .../TokenEndpointAuthMethodField.tsx | 83 +- .../_components/TokenExchangeFormFields.tsx | 74 +- .../_components/ToolArgumentsForm.tsx | 68 +- .../mcp-servers/_components/ToolTestPanel.tsx | 76 +- .../_components/TruePassthroughWarning.tsx | 11 +- .../_components/UpstreamTokenHeaderField.tsx | 46 +- .../_components/UserEnvVarsModal.tsx | 67 +- .../_components/duplicateServerCheck.ts | 3 +- .../_components/editServerPayload.ts | 18 +- .../_components/editToolPreview.ts | 6 +- .../_components/importConnectorConfig.ts | 21 +- .../mcp-servers/_components/mcpFieldRules.ts | 7 +- .../mcp-servers/_components/mcp_connect.tsx | 245 ++- .../_components/mcp_connection_status.tsx | 42 +- .../mcp-servers/_components/mcp_discovery.tsx | 28 +- .../_components/mcp_server_cost_config.tsx | 49 +- .../_components/mcp_server_cost_display.tsx | 24 +- .../_components/mcp_server_edit.tsx | 194 +- .../_components/mcp_server_view.tsx | 122 +- .../mcp-servers/_components/mcp_servers.tsx | 121 +- .../_components/mcp_tool_configuration.tsx | 90 +- .../mcp-servers/_components/mcp_tools.tsx | 66 +- .../_components/toolCallArguments.ts | 22 +- .../mcp-servers/_components/utils.tsx | 34 +- .../src/app/mcp/oauth/callback/page.tsx | 15 +- .../MCPSemanticFilterSettings.tsx | 82 +- .../MCPSemanticFilterTestPanel.tsx | 67 +- .../semanticFilterTestUtils.ts | 46 +- .../MCPToolSearchSettings.tsx | 51 +- .../MCPServerSelector.tsx | 24 +- .../MCPToolPermissions.tsx | 91 +- .../mcp_tools/ByokCredentialModal.tsx | 64 +- .../mcp_tools/McpCrudPermissionPanel.tsx | 27 +- .../src/hooks/useMcpOAuthFlow.tsx | 18 +- .../src/hooks/useTestMCPConnection.tsx | 4 +- .../src/hooks/useToolsOAuthFlow.tsx | 8 +- ui/litellm-dashboard/src/i18n/locales/en.json | 1768 +++++++++++++++++ 60 files changed, 4151 insertions(+), 1630 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx index 5f64d003c11..b153d0a6456 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx @@ -1,5 +1,6 @@ import { Info } from "lucide-react"; import React from "react"; +import { useTranslation } from "react-i18next"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { MountedFormField } from "@/components/common_components/MountedFormField"; @@ -22,134 +23,150 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt const ACCESS_KEY_PATH = ["credentials", "aws_access_key_id"] as const; const SECRET_KEY_PATH = ["credentials", "aws_secret_access_key"] as const; -const AwsSigV4Fields: React.FC = () => ( - <> -

- For MCP servers hosted on AWS Bedrock AgentCore.{" "} - { + const { t } = useTranslation(); + return ( + <> +

+ {t("mcpServers.awsSigV4Fields.intro")}{" "} + + {t("mcpServers.awsSigV4Fields.viewDocs")} + +

+ + } + name={["credentials", "aws_region_name"]} + required + rules={{ validate: { required: requiredRule(t("mcpServers.awsSigV4Fields.region.required")) } }} > - View docs → - -

- } - name={["credentials", "aws_region_name"]} - required - rules={{ validate: { required: requiredRule("AWS region is required for SigV4 auth") } }} - > - {(control) => } - - - } - name={["credentials", "aws_service_name"]} - > - {(control) => } - - - } - name={ACCESS_KEY_PATH} - rules={{ - deps: ["credentials.aws_secret_access_key"], - validate: { - pairedWithSecret: requiredWhenSiblingSet( - SECRET_KEY_PATH, - "Access Key ID is required when Secret Access Key is provided", - ), - }, - }} - > - {(control) => ( - - )} - - - } - name={SECRET_KEY_PATH} - rules={{ - deps: ["credentials.aws_access_key_id"], - validate: { - pairedWithAccessKey: requiredWhenSiblingSet( - ACCESS_KEY_PATH, - "Secret Access Key is required when Access Key ID is provided", - ), - }, - }} - > - {(control) => ( - - )} - - } - name={["credentials", "aws_session_token"]} - > - {(control) => ( - - )} - - - } - name={["credentials", "aws_role_name"]} - > - {(control) => ( - - )} - - - } - name={["credentials", "aws_session_name"]} - > - {(control) => ( - - )} - - -); + {(control) => } +
+ + } + name={["credentials", "aws_service_name"]} + > + {(control) => } + + + } + name={ACCESS_KEY_PATH} + rules={{ + deps: ["credentials.aws_secret_access_key"], + validate: { + pairedWithSecret: requiredWhenSiblingSet( + SECRET_KEY_PATH, + t("mcpServers.awsSigV4Fields.accessKeyId.requiredWithSecret"), + ), + }, + }} + > + {(control) => ( + + )} + + + } + name={SECRET_KEY_PATH} + rules={{ + deps: ["credentials.aws_access_key_id"], + validate: { + pairedWithAccessKey: requiredWhenSiblingSet( + ACCESS_KEY_PATH, + t("mcpServers.awsSigV4Fields.secretAccessKey.requiredWithAccessKey"), + ), + }, + }} + > + {(control) => ( + + )} + + + } + name={["credentials", "aws_session_token"]} + > + {(control) => ( + + )} + + + } + name={["credentials", "aws_role_name"]} + > + {(control) => ( + + )} + + + } + name={["credentials", "aws_session_name"]} + > + {(control) => ( + + )} + + + ); +}; export default AwsSigV4Fields; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx index 07618abb9a4..14a6e3d941f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx @@ -1,5 +1,7 @@ +import type { TFunction } from "i18next"; import React, { useState } from "react"; import { FormProvider, useForm, useWatch } from "react-hook-form"; +import { useTranslation } from "react-i18next"; import { ChevronDown, Info } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; @@ -35,7 +37,7 @@ import { buildCreateServerPayload, reduceStaticHeaders, } from "./createServerPayload"; -import { DUPLICATE_IDENTIFIER_MESSAGE, findDuplicateMcpServer, mcpSubmitErrorReason } from "./duplicateServerCheck"; +import { DUPLICATE_IDENTIFIER_MESSAGE_KEY, findDuplicateMcpServer, mcpSubmitErrorReason } from "./duplicateServerCheck"; import { readCreateUiSnapshot, writeCreateUiSnapshot } from "./createOAuthUiState"; import AwsSigV4Fields from "./AwsSigV4Fields"; import OpenApiByokFields from "./OpenApiByokFields"; @@ -86,14 +88,14 @@ interface CreateMCPServerProps { stdioEnabled?: boolean; } -const payloadErrorMessage = (result: Exclude): string => { +const payloadErrorMessage = (result: Exclude, t: TFunction): string => { switch (result.kind) { case "invalid_tool_display_name": - return `Tool display name "${result.displayName}" is invalid. Only letters, digits, underscores, and hyphens are allowed (no spaces).`; + return t("mcpServers.createMcpServer.payloadErrors.invalidToolDisplayName", { displayName: result.displayName }); case "invalid_stdio_json": - return "Invalid JSON in stdio configuration"; + return t("mcpServers.createMcpServer.payloadErrors.invalidStdioJson"); case "invalid_token_validation_json": - return "Invalid JSON in Token Validation Rules"; + return t("mcpServers.createMcpServer.payloadErrors.invalidTokenValidationJson"); } }; @@ -117,6 +119,7 @@ const CreateMCPServer: React.FC = ({ onBackToDiscovery, stdioEnabled = true, }) => { + const { t } = useTranslation(); const form = useForm({ mode: "onChange", defaultValues: CREATE_DEFAULTS }); const registry = useMountRegistry(); const [isLoading, setIsLoading] = useState(false); @@ -255,9 +258,7 @@ const CreateMCPServer: React.FC = ({ // which would persist it as server-level credentials on the created server row. Mirrors the // edit form's onTokenReceived early return. setAuthorizedIdentity(getOAuthAuthorizationIdentity(allFieldsValue(form))); - toast.success( - "Token held for this browser session. Tools can now be previewed and configured; the token is not saved to LiteLLM.", - ); + toast.success(t("mcpServers.createMcpServer.toasts.browserTokenHeld")); return; } @@ -283,7 +284,7 @@ const CreateMCPServer: React.FC = ({ // its own credential write. setAuthorizedIdentity(getOAuthAuthorizationIdentity(allFieldsValue(form))); - toast.success("OAuth authorization successful! Please click 'Create MCP Server' to save the configuration."); + toast.success(t("mcpServers.createMcpServer.toasts.oauthAuthorized")); }, onBeforeRedirect: persistCreateUiState, flowSource: "create", @@ -425,8 +426,8 @@ const CreateMCPServer: React.FC = ({ typeof values.alias === "string" ? values.alias : undefined, ); if (duplicate) { - form.setError(duplicate.field, { type: "duplicate", message: DUPLICATE_IDENTIFIER_MESSAGE }); - toast.fromError(DUPLICATE_IDENTIFIER_MESSAGE); + form.setError(duplicate.field, { type: "duplicate", message: t(DUPLICATE_IDENTIFIER_MESSAGE_KEY) }); + toast.fromError(t(DUPLICATE_IDENTIFIER_MESSAGE_KEY)); return; } const built = buildCreateServerPayload(values, { @@ -440,7 +441,7 @@ const CreateMCPServer: React.FC = ({ dcrClient: dcrClientRef.current, }); if (built.kind !== "ok") { - toast.fromError(payloadErrorMessage(built)); + toast.fromError(payloadErrorMessage(built, t)); return; } const payload = built.payload; @@ -482,10 +483,10 @@ const CreateMCPServer: React.FC = ({ } if (isAdmin) { - toast.success("MCP Server created successfully"); + toast.success(t("mcpServers.createMcpServer.toasts.created")); } else { - toast.success("MCP Server submitted for admin review", { - description: "Once an admin approves it, the server will appear in your MCP Servers list.", + toast.success(t("mcpServers.createMcpServer.toasts.submittedForReview"), { + description: t("mcpServers.createMcpServer.toasts.submittedForReviewDescription"), }); } form.reset(CREATE_DEFAULTS); @@ -500,7 +501,11 @@ const CreateMCPServer: React.FC = ({ } } catch (error) { const reason = mcpSubmitErrorReason(error); - toast.fromError(isAdmin ? `Error creating MCP Server: ${reason}` : `Error submitting MCP Server: ${reason}`); + toast.fromError( + isAdmin + ? t("mcpServers.createMcpServer.toasts.createFailed", { reason }) + : t("mcpServers.createMcpServer.toasts.submitFailed", { reason }), + ); } finally { setIsLoading(false); } @@ -640,9 +645,11 @@ const CreateMCPServer: React.FC = ({ ← )} - MCP Logo + {t("mcpServers.createMcpServer.logoAlt")} - {isAdmin ? "Add New MCP Server" : "Submit MCP Server for Review"} + {isAdmin + ? t("mcpServers.createMcpServer.title.admin") + : t("mcpServers.createMcpServer.title.submitForReview")} @@ -653,27 +660,26 @@ const CreateMCPServer: React.FC = ({
{!isAdmin && (
- Your submission will be sent for admin review. Once approved, the server will appear in your MCP - Servers list. The request must be made with a team-scoped API key. + {t("mcpServers.createMcpServer.reviewNotice")}
)}
- MCP Server Name - + {t("mcpServers.createMcpServer.serverName.label")} + } name="server_name" - rules={{ validate: validatorRules({ validator: (_, value) => validateMCPServerName(value) }) }} + rules={{ validate: validatorRules({ validator: (_, value) => validateMCPServerName(value, t) }) }} > {(control) => ( )} @@ -682,19 +688,19 @@ const CreateMCPServer: React.FC = ({ - Alias - + {t("mcpServers.createMcpServer.alias.label")} + } name="alias" - rules={{ validate: validatorRules({ validator: (_, value) => validateMCPServerName(value) }) }} + rules={{ validate: validatorRules({ validator: (_, value) => validateMCPServerName(value, t) }) }} > {(control) => ( { control.onChange(event); @@ -705,13 +711,17 @@ const CreateMCPServer: React.FC = ({ Description} + label={ + + {t("mcpServers.createMcpServer.description.label")} + + } name="description" > {(control) => ( )} @@ -720,7 +730,11 @@ const CreateMCPServer: React.FC = ({ GitHub / Source URL} + label={ + + {t("mcpServers.createMcpServer.sourceUrl.label")} + + } name="source_url" > {(control) => ( @@ -733,10 +747,14 @@ const CreateMCPServer: React.FC = ({ Transport Type} + label={ + + {t("mcpServers.createMcpServer.transport.label")} + + } name="transport" required - rules={{ validate: { required: requiredRule("Please select a transport type") } }} + rules={{ validate: { required: requiredRule(t("mcpServers.createMcpServer.transport.required")) } }} > {(control) => ( (control)} items={AUTH_TYPE_ITEMS}> - + {AUTH_TYPE_ITEMS.map((item) => ( @@ -886,8 +912,8 @@ const CreateMCPServer: React.FC = ({ - Authentication Value - + {t("mcpServers.createMcpServer.auth.valueLabel")} + @@ -895,14 +921,14 @@ const CreateMCPServer: React.FC = ({ name={["credentials", "auth_value"]} rules={{ validate: { - notWhitespace: notOnlyWhitespace("Authentication value cannot be empty whitespace"), + notWhitespace: notOnlyWhitespace(t("mcpServers.createMcpServer.auth.valueWhitespace")), }, }} > {(control) => ( )} @@ -1000,11 +1026,13 @@ const CreateMCPServer: React.FC = ({
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx index 893981fa9c6..48a8f4d7a64 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx @@ -1,5 +1,6 @@ import { Info } from "lucide-react"; import React from "react"; +import { useTranslation } from "react-i18next"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { Switch } from "@/components/ui/switch"; @@ -23,13 +24,14 @@ export default function DcrBridgeToggle({ authType?: string | null; initialChecked?: boolean; }) { + const { t } = useTranslation(); if (!isClientForwardedTokenMode(authType)) return null; return ( - Gateway-hosted sign-in (DCR bridge) - + {t("mcpServers.dcrBridgeToggle.label")} + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx index 378410e4790..a804d4697c4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx @@ -1,5 +1,7 @@ +import type { ParseKeys } from "i18next"; import { CircleMinus, Info, Plus } from "lucide-react"; import React from "react"; +import { Trans, useTranslation } from "react-i18next"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { Button } from "@/components/ui/button"; @@ -16,11 +18,13 @@ import { requiredRule } from "@/components/common_components/formRules"; import { matchesPattern, selectControl, selectTriggerControl, textControl } from "./mcpFieldRules"; import { listControl } from "./mcpFormStore"; -const SCOPE_OPTIONS = [ - { value: "global", label: "Instance" }, - { value: "user", label: "Per-user" }, +const SCOPE_OPTIONS: ReadonlyArray<{ value: string; labelKey: ParseKeys }> = [ + { value: "global", labelKey: "mcpServers.envVarsSection.scopes.instance" }, + { value: "user", labelKey: "mcpServers.envVarsSection.scopes.perUser" }, ]; +const VARIABLE_SYNTAX = "${VAR_NAME}"; + const VARIABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; /** @@ -34,31 +38,34 @@ const VARIABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; * The parent form reads the ``env_vars`` field from the form values. */ const EnvVarsSection: React.FC = () => { + const { t } = useTranslation(); const { control } = useFormContext(); const { fields, append, remove } = useFieldArray({ control: listControl(control), name: "env_vars" }); useMountedName("env_vars"); + const scopeOptions = SCOPE_OPTIONS.map((option) => ({ value: option.value, label: t(option.labelKey) })); return (
- Variables + {t("mcpServers.envVarsSection.title")} - Define variables you can interpolate in Static Headers or Authentication using{" "} - {"${VAR_NAME}"}.
- Instance: admin-defined value used for every user. -
- Per-user: each user supplies their own value (e.g. personal credentials) via the MCP Gateway - dashboard. - + , b: }} + /> } >
- Reference these in Static Headers or Authentication as {"${VAR_NAME}"}. For example:{" "} + }} + />{" "} {"${DB_PROTOCOL}://${CORP_USERNAME}:${CORP_PASSWORD}@${DB_HOSTNAME}"} @@ -67,9 +74,9 @@ const EnvVarsSection: React.FC = () => {
{fields.length > 0 && (
-
Variable Name
-
Value / Description
-
Scope
+
{t("mcpServers.envVarsSection.columns.name")}
+
{t("mcpServers.envVarsSection.columns.value")}
+
{t("mcpServers.envVarsSection.columns.scope")}
)} @@ -80,16 +87,17 @@ const EnvVarsSection: React.FC = () => { className="mb-0 flex-1" rules={{ validate: { - required: requiredRule("Variable name is required"), - pattern: matchesPattern( - VARIABLE_NAME_PATTERN, - "Use letters, digits, underscores; cannot start with a digit.", - ), + required: requiredRule(t("mcpServers.envVarsSection.name.required")), + pattern: matchesPattern(VARIABLE_NAME_PATTERN, t("mcpServers.envVarsSection.name.invalid")), }, }} > {(control) => ( - + )}
@@ -97,12 +105,12 @@ const EnvVarsSection: React.FC = () => {
{(control) => ( - (control)} items={scopeOptions}> - {SCOPE_OPTIONS.map((option) => ( + {scopeOptions.map((option) => ( {option.label} @@ -121,7 +129,7 @@ const EnvVarsSection: React.FC = () => { ))}
@@ -132,6 +140,7 @@ const EnvVarsSection: React.FC = () => { // vars the value comes from each user later, so the column instead captures an // optional description that the per-user fill-in modal shows as a hint. const ScopedValueOrDescription: React.FC<{ index: number }> = ({ index }) => { + const { t } = useTranslation(); const isPerUser = useWatch({ name: `env_vars.${index}.scope` }) === "user"; if (isPerUser) { return ( @@ -139,16 +148,16 @@ const ScopedValueOrDescription: React.FC<{ index: number }> = ({ index }) => { {(control) => ( - + - Hint + {t("mcpServers.envVarsSection.hint.label")} @@ -158,7 +167,13 @@ const ScopedValueOrDescription: React.FC<{ index: number }> = ({ index }) => { } return ( - {(control) => } + {(control) => ( + + )} ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx index 70d1bc40c18..98bb3254ba5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx @@ -1,5 +1,6 @@ import { Info } from "lucide-react"; import React from "react"; +import { useTranslation } from "react-i18next"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { MountedFormField } from "@/components/common_components/MountedFormField"; @@ -27,9 +28,10 @@ const FieldLabel: React.FC<{ label: string; tooltip: string }> = ({ label, toolt ); const PRIVATE_KEY_PATH = ["credentials", "client_private_key"] as const; +const PEM_HEADER = "-----BEGIN PRIVATE KEY-----"; const IdJagFormFields: React.FC = ({ isEditing = false }) => { - const placeholderSuffix = isEditing ? " (leave blank to keep existing)" : ""; + const { t } = useTranslation(); const requiredWhenCreating = (message: string) => isEditing ? undefined : { validate: { required: requiredRule(message) } }; @@ -38,13 +40,13 @@ const IdJagFormFields: React.FC = ({ isEditing = false }) } name="token_exchange_endpoint" required={!isEditing} - rules={requiredWhenCreating("The org token endpoint is required for ID-JAG")} + rules={requiredWhenCreating(t("mcpServers.idJagFormFields.orgTokenEndpoint.required"))} > {(control) => ( = ({ isEditing = false }) } name={["credentials", "id_jag_resource_token_endpoint"]} required={!isEditing} - rules={requiredWhenCreating("The resource token endpoint is required for ID-JAG")} + rules={requiredWhenCreating(t("mcpServers.idJagFormFields.resourceTokenEndpoint.required"))} > {(control) => ( = ({ isEditing = false }) )} } + label={ + + } name={["credentials", "client_id"]} required={!isEditing} - rules={requiredWhenCreating("Client ID is required for ID-JAG")} + rules={requiredWhenCreating(t("mcpServers.idJagFormFields.clientId.required"))} > {(control) => ( )} @@ -90,8 +101,8 @@ const IdJagFormFields: React.FC = ({ isEditing = false }) } name={["credentials", "client_secret"]} @@ -103,7 +114,7 @@ const IdJagFormFields: React.FC = ({ isEditing = false }) validate: { secretOrPrivateKey: requiredUnlessSiblingSet( PRIVATE_KEY_PATH, - "Provide either a client secret or a client private key", + t("mcpServers.idJagFormFields.clientSecret.secretOrPrivateKey"), ), }, } @@ -112,7 +123,11 @@ const IdJagFormFields: React.FC = ({ isEditing = false }) {(control) => ( )} @@ -120,8 +135,8 @@ const IdJagFormFields: React.FC = ({ isEditing = false }) } name={PRIVATE_KEY_PATH} @@ -130,7 +145,11 @@ const IdJagFormFields: React.FC = ({ isEditing = false })