From 5107f205a06d5495731b85962b9bd823a33ec158 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:46:59 -0700 Subject: [PATCH 001/171] refactor: clean up fresh tech debt from 2026-09-30 (#43993) * refactor: clean up fresh tech debt from 2026-09-30 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor: keep agent tracing route comment in LiteLLMRoutes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor: move agent tracing route comment above the trace routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor: drop route comment that duplicates the trace handler docstring Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 - litellm/proxy/_types.py | 1 - litellm/proxy/roi_calculator/estimator.py | 26 ---------------------- litellm/proxy/roi_calculator/pull_cache.py | 5 ----- litellm/tracing/decode.py | 6 ----- litellm/tracing/receiver.py | 4 ---- 6 files changed, 43 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index a8c6278c62f..b3f5b0471f4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -46,7 +46,6 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( ) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) -# Agent tracing / ClickHouse CLICKHOUSE_BATCH_SIZE: Final = get_env_int("CLICKHOUSE_BATCH_SIZE", 10_000) CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = float(os.getenv("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", "1.0")) CLICKHOUSE_MAX_BUFFERED_ROWS: Final = get_env_int("CLICKHOUSE_MAX_BUFFERED_ROWS", 200_000) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7b1ba2ec1ac..649381f24a2 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -520,7 +520,6 @@ class LiteLLMRoutes(enum.Enum): "/v1/rag/ingest", "/rag/query", "/v1/rag/query", - # agent tracing: OTLP ingest + reads (scoped to the caller's team in the handler) "/engine", "/engine/{engine_id}", "/engine/{engine_id}/runs", diff --git a/litellm/proxy/roi_calculator/estimator.py b/litellm/proxy/roi_calculator/estimator.py index 200cc38f5c6..4cb211f9cb0 100644 --- a/litellm/proxy/roi_calculator/estimator.py +++ b/litellm/proxy/roi_calculator/estimator.py @@ -92,32 +92,6 @@ def cache_context(settings: ROISettings, models: tuple[EstimatorModel, ...] | No return hashlib.sha256(context.encode()).hexdigest() -def pull_cache_key( - settings: ROISettings, - pull: ROIPullEvidence, - models: tuple[EstimatorModel, ...] | None = None, -) -> str: - evidence: Final = json.dumps( - metadata_evidence(pull).model_dump(exclude_unset=True), - ensure_ascii=False, - ) - key: Final = json.dumps( - ( - ESTIMATE_VERSION, - settings.estimator_model, - settings.estimator_prompt, - RESPONSE_CONTRACT, - estimator_options(_configured_models(settings, models)), - pull["repo"], - pull["number"], - pull["head_sha"], - evidence, - ), - ensure_ascii=False, - ) - return hashlib.sha256(key.encode()).hexdigest() - - class Estimator: def __init__( self, diff --git a/litellm/proxy/roi_calculator/pull_cache.py b/litellm/proxy/roi_calculator/pull_cache.py index d82b0d60f14..e1800fd0620 100644 --- a/litellm/proxy/roi_calculator/pull_cache.py +++ b/litellm/proxy/roi_calculator/pull_cache.py @@ -2,7 +2,6 @@ import hashlib import json from typing import Final -from litellm.proxy.roi_calculator.estimator import cache_context from litellm.proxy.roi_calculator.github import GitHubPullListItem from litellm.types.roi_calculator import ROISettings @@ -46,7 +45,3 @@ def settings_fingerprint(settings: ROISettings) -> str: ensure_ascii=False, ) return hashlib.sha256(value.encode()).hexdigest() - - -def current_cache_context(settings: ROISettings) -> str: - return cache_context(settings) diff --git a/litellm/tracing/decode.py b/litellm/tracing/decode.py index 60ae7732444..c310a339593 100644 --- a/litellm/tracing/decode.py +++ b/litellm/tracing/decode.py @@ -52,9 +52,6 @@ class OTLPPayloadTooLargeError(OverflowError): pass -# ---------------------------------------------------------------- decode - - def _truncate(value: str) -> str: size = len(value.encode("utf-8")) if size <= OTLP_MAX_ATTRIBUTE_VALUE_BYTES: @@ -123,9 +120,6 @@ def _span_row(span: DecodedSpan) -> SpanRow: return row -# ---------------------------------------------------------------- normalize - - def _loads(value: str) -> object: try: return json.loads(value) diff --git a/litellm/tracing/receiver.py b/litellm/tracing/receiver.py index be9641a602b..8b157e260a8 100644 --- a/litellm/tracing/receiver.py +++ b/litellm/tracing/receiver.py @@ -82,8 +82,6 @@ class TraceReceiver: spend_log_retention_days=AGENT_TRACING_SPEND_LOG_RETENTION_DAYS, ) - # ------------------------------------------------------------ write - async def ingest( self, body: bytes, @@ -108,8 +106,6 @@ class TraceReceiver: raise TracingPayloadTooLargeError(str(error)) from error return len(rows) - # ------------------------------------------------------------ read - async def list_traces(self, scope: TraceScope, start_ms: int, end_ms: int, cursor: str | None = None) -> TracePage: return await self.store.list_traces(scope, start_ms, end_ms, cursor) From b4bb2a77a20231860d91220376a1ec2691481b8b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 05:16:49 -0700 Subject: [PATCH 002/171] test: inject the HIBP client and the MCP loop clock so two backend tests stop flaking (#44007) * test(proxy): inject the HIBP client into the breached-password update test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(mcp): drive optional-discovery deadlines with an injected loop clock Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: bound MCP deadline checks, support Python 3.10, pin HIBP URL Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../internal_user_endpoints.py | 15 +- .../test_internal_user_endpoints.py | 17 +- .../test_mcp_client.py | 172 +++++++++++------- 3 files changed, 126 insertions(+), 78 deletions(-) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index ee1ebcb5ce9..4176f57d9de 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -181,7 +181,10 @@ def _team_membership_table( async def _hash_password_in_dict( - data: dict, general_settings: Mapping[str, object], password_prevalidated: bool = False + data: dict, + general_settings: Mapping[str, object], + password_prevalidated: bool = False, + hibp_client: AsyncHTTPHandler | None = None, ) -> None: """Validate and hash password field in-place if present. @@ -193,7 +196,7 @@ async def _hash_password_in_dict( if "password" in data and data["password"] is not None: if not password_prevalidated: validate_password_policy(data["password"], general_settings) - await validate_password_not_breached(data["password"], general_settings) + await validate_password_not_breached(data["password"], general_settings, hibp_client) data["password"] = hash_password(data["password"]) data["password_reset_required"] = True data["last_breach_check_at"] = None @@ -1459,6 +1462,7 @@ async def _update_single_user_helper( user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None = None, password_prevalidated: bool = False, + hibp_client: AsyncHTTPHandler | None = None, ) -> dict[str, Any]: """ Helper function to update a single user. @@ -1481,7 +1485,12 @@ async def _update_single_user_helper( data_json: Final[dict] = user_request.model_dump(exclude_unset=True) non_default_values = _update_internal_user_params(data_json=data_json, data=user_request) - await _hash_password_in_dict(non_default_values, general_settings, password_prevalidated=password_prevalidated) + await _hash_password_in_dict( + non_default_values, + general_settings, + password_prevalidated=password_prevalidated, + hibp_client=hibp_client, + ) existing_user_row: BaseModel | None = None if user_request.user_id: diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index c663e63414c..adcdfea4711 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -10,7 +10,6 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest -import respx from fastapi import HTTPException from fastapi.testclient import TestClient from pytest_mock import MockerFixture @@ -4547,7 +4546,6 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo @pytest.mark.asyncio -@respx.mock async def test_user_update_rejects_breached_password(_admin_prisma): """A strength-passing password found in the HIBP corpus must be rejected before it ever reaches the DB write.""" @@ -4557,19 +4555,26 @@ async def test_user_update_rejects_breached_password(_admin_prisma): password = "Str0ng!Passw0rd" sha1 = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() - respx.get(f"https://api.pwnedpasswords.com/range/{sha1[:5]}").mock( - return_value=httpx.Response(200, text=f"{sha1[5:]}:1387") - ) + lookups: Final[list[tuple[str, str]]] = [] # mutable-ok: capture the injected handler request method and URL + + def handler(request: httpx.Request) -> httpx.Response: + lookups.append((request.method, str(request.url))) + return httpx.Response(200, text=f"{sha1[5:]}:1387") user_request = UpdateUserRequest(user_id="target-user", password=password) admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) with pytest.raises(ProxyException) as exc_info: - await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + await _update_single_user_helper( + user_request=user_request, + user_api_key_dict=admin_caller, + hibp_client=_hibp_client_with_handler(handler), + ) assert exc_info.value.code == "400" assert "data breaches" in exc_info.value.message _admin_prisma.db.litellm_usertable.find_first.assert_not_called() + assert lookups == [("GET", f"https://api.pwnedpasswords.com/range/{sha1[:5]}")] @pytest.mark.asyncio diff --git a/tests/unit/experimental_mcp_client/test_mcp_client.py b/tests/unit/experimental_mcp_client/test_mcp_client.py index 1a56227b008..504219a64e1 100644 --- a/tests/unit/experimental_mcp_client/test_mcp_client.py +++ b/tests/unit/experimental_mcp_client/test_mcp_client.py @@ -81,6 +81,20 @@ class _MockTransportClient(MCPClient): return streamable_http_client(self.server_url, http_client=http_client), http_client +class _ManualClockLoop(asyncio.SelectorEventLoop): + """An event loop whose clock moves only when the test advances it, so timeouts fire on test-controlled conditions""" + + def __init__(self) -> None: + super().__init__() + self._now = 0.0 + + def time(self) -> float: + return self._now + + def advance(self, seconds: float) -> None: + self._now += seconds + + class _FakeExceptionGroup(Exception): """Duck-typed stand-in for an anyio/builtin ExceptionGroup. @@ -2309,16 +2323,20 @@ async def test_optional_discovery_collects_all_pages(method: str, session_id: st assert sum(call.args[0].method == "DELETE" for call in responder.call_args_list) == (1 if session_id else 0) -@pytest.mark.asyncio @pytest.mark.parametrize("method", ("prompts/list", "resources/list", "resources/templates/list")) @pytest.mark.parametrize( "failure", ("repeat", "cycle", "cap", "method_not_found", "internal_error", "unauthorized", "deadline") ) @pytest.mark.parametrize("strict", (False, True)) -async def test_optional_discovery_rejects_incomplete_walks( +def test_optional_discovery_rejects_incomplete_walks( method: str, failure: str, strict: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - monkeypatch.setattr(mcp_client_module, "MCP_TOOL_LISTING_MAX_PAGES", 3 if failure == "cycle" else 2, raising=False) + monkeypatch.setattr( + mcp_client_module, + "MCP_TOOL_LISTING_MAX_PAGES", + 3 if failure in ("cycle", "repeat") else 2, + raising=False, + ) monkeypatch.setattr(mcp_client_module, "MCP_TOOL_LISTING_TIMEOUT", 0.05) field: Final = { "prompts/list": "prompts", @@ -2330,82 +2348,98 @@ async def test_optional_discovery_rejects_incomplete_walks( "resources/list": {"name": "first", "uri": "test://first"}, "resources/templates/list": {"name": "first", "uriTemplate": "test://{name}"}, }[method] - cancelled: Final = asyncio.Event() + loop: Final = _ManualClockLoop() - async def respond(request: httpx2.Request) -> httpx2.Response: - payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) - if not isinstance(payload, JSONRPCRequest): - return httpx2.Response(202) - if payload.method == "initialize": - return httpx2.Response( - 200, - json={ - "jsonrpc": "2.0", - "id": payload.id, - "result": { - "protocolVersion": (payload.params or {})["protocolVersion"], - "capabilities": {"prompts": {}, "resources": {}}, - "serverInfo": {"name": "interrupted", "version": "1"}, - }, - }, - ) - assert payload.method == method - cursor: Final = (payload.params or {}).get("cursor") - if cursor is not None: - if failure == "deadline": - try: - await asyncio.Event().wait() - finally: - cancelled.set() - if failure == "unauthorized": - return httpx2.Response(401) - if failure in ("method_not_found", "internal_error"): + async def run() -> None: + cancelled: Final = asyncio.Event() + + async def respond(request: httpx2.Request) -> httpx2.Response: + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + if not isinstance(payload, JSONRPCRequest): + return httpx2.Response(202) + if payload.method == "initialize": return httpx2.Response( 200, json={ "jsonrpc": "2.0", "id": payload.id, - "error": { - "code": -32601 if failure == "method_not_found" else -32603, - "message": "Later page unavailable", + "result": { + "protocolVersion": (payload.params or {})["protocolVersion"], + "capabilities": {"prompts": {}, "resources": {}}, + "serverInfo": {"name": "interrupted", "version": "1"}, }, }, ) - next_cursor: Final = ( - "private-cursor-2" if cursor == "private-cursor-1" and failure != "repeat" else "private-cursor-1" - ) - return httpx2.Response( - 200, json={"jsonrpc": "2.0", "id": payload.id, "result": {field: [entry], "nextCursor": next_cursor}} - ) + assert payload.method == method + cursor: Final = (payload.params or {}).get("cursor") + if cursor is not None: + if failure == "deadline": + loop.advance(0.15) + try: + for _ in range(1_000): + await asyncio.sleep(0) + except asyncio.CancelledError: + cancelled.set() + raise + return httpx2.Response(500) + if failure == "unauthorized": + return httpx2.Response(401) + if failure in ("method_not_found", "internal_error"): + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "error": { + "code": -32601 if failure == "method_not_found" else -32603, + "message": "Later page unavailable", + }, + }, + ) + if failure == "deadline" and cursor is None: + loop.advance(0.1) + next_cursor: Final = ( + "private-cursor-2" if cursor == "private-cursor-1" and failure != "repeat" else "private-cursor-1" + ) + return httpx2.Response( + 200, json={"jsonrpc": "2.0", "id": payload.id, "result": {field: [entry], "nextCursor": next_cursor}} + ) - responder: Final = AsyncMock(side_effect=respond) - client: Final = _MockTransportClient(responder, server_url="https://example.com/mcp", timeout=0.2) - operation: Final = { - "prompts/list": client.list_prompts, - "resources/list": client.list_resources, - "resources/templates/list": client.list_resource_templates, - }[method] - if strict: - error_type: Final = { - "internal_error": MCPError, - "unauthorized": httpx2.HTTPStatusError, - "deadline": TimeoutError, - }.get(failure, RuntimeError) - with pytest.raises(error_type): - await operation(raise_on_error=True) - else: - assert await operation() == [] - assert len( - tuple( - payload - for call in responder.call_args_list - if isinstance(payload := _JSONRPC_MESSAGE_ADAPTER.validate_json(call.args[0].content), JSONRPCRequest) - and payload.method == method - ) - ) == (3 if failure == "cycle" else 2) - assert "private-cursor" not in caplog.text - if failure == "deadline": - assert cancelled.is_set() + responder: Final = AsyncMock(side_effect=respond) + client: Final = _MockTransportClient(responder, server_url="https://example.com/mcp", timeout=0.2) + operation: Final = { + "prompts/list": client.list_prompts, + "resources/list": client.list_resources, + "resources/templates/list": client.list_resource_templates, + }[method] + if strict: + error_type: Final = { + "internal_error": MCPError, + "unauthorized": httpx2.HTTPStatusError, + "deadline": TimeoutError, + }.get(failure, RuntimeError) + with pytest.raises(error_type): + await operation(raise_on_error=True) + else: + assert await operation() == [] + assert len( + tuple( + payload + for call in responder.call_args_list + if isinstance(payload := _JSONRPC_MESSAGE_ADAPTER.validate_json(call.args[0].content), JSONRPCRequest) + and payload.method == method + ) + ) == (3 if failure == "cycle" else 2) + assert "private-cursor" not in caplog.text + if failure == "deadline": + assert cancelled.is_set() + + try: + loop.run_until_complete(run()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) + loop.close() @pytest.mark.asyncio From 3a11192f6898749c380390d2b68ae00c8581791e Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:16:30 -0700 Subject: [PATCH 003/171] fix(cost-map): reprice fireworks deepseek v4.1 flash to the 2026-10-01 pricing update (#44024) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 28 +++++++++---------- model_prices_and_context_window.json | 28 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2a01c4fe862..44b5cb0f59f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -61053,18 +61053,18 @@ "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "cache_read_input_token_cost": 7e-09, - "cache_read_input_token_cost_priority": 8.75e-09, - "input_cost_per_token": 2.2e-07, - "input_cost_per_token_priority": 2.75e-07, + "cache_read_input_token_cost": 6e-09, + "cache_read_input_token_cost_priority": 7.5e-09, + "input_cost_per_token": 3e-07, + "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 6.6e-07, - "output_cost_per_token_priority": 8.25e-07, - "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_priority": 1.5e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -61155,18 +61155,18 @@ "supports_vision": false }, "fireworks_ai/deepseek-v4p1-flash": { - "cache_read_input_token_cost": 7e-09, - "cache_read_input_token_cost_priority": 8.75e-09, - "input_cost_per_token": 2.2e-07, - "input_cost_per_token_priority": 2.75e-07, + "cache_read_input_token_cost": 6e-09, + "cache_read_input_token_cost_priority": 7.5e-09, + "input_cost_per_token": 3e-07, + "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 6.6e-07, - "output_cost_per_token_priority": 8.25e-07, - "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_priority": 1.5e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2a01c4fe862..44b5cb0f59f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -61053,18 +61053,18 @@ "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "cache_read_input_token_cost": 7e-09, - "cache_read_input_token_cost_priority": 8.75e-09, - "input_cost_per_token": 2.2e-07, - "input_cost_per_token_priority": 2.75e-07, + "cache_read_input_token_cost": 6e-09, + "cache_read_input_token_cost_priority": 7.5e-09, + "input_cost_per_token": 3e-07, + "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 6.6e-07, - "output_cost_per_token_priority": 8.25e-07, - "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_priority": 1.5e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -61155,18 +61155,18 @@ "supports_vision": false }, "fireworks_ai/deepseek-v4p1-flash": { - "cache_read_input_token_cost": 7e-09, - "cache_read_input_token_cost_priority": 8.75e-09, - "input_cost_per_token": 2.2e-07, - "input_cost_per_token_priority": 2.75e-07, + "cache_read_input_token_cost": 6e-09, + "cache_read_input_token_cost_priority": 7.5e-09, + "input_cost_per_token": 3e-07, + "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 6.6e-07, - "output_cost_per_token_priority": 8.25e-07, - "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_priority": 1.5e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, From 4b06d043347f71bbeceab996cd46f2739ea21d55 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:12:53 -0700 Subject: [PATCH 004/171] test(anthropic): native /v1/messages reasoning integration tests built on a captured Claude Code request (#43361) * test(integration): group /v1/messages contracts under tests/integration/messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): make ci coverage census collect nested test dirs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): nest /v1/messages contracts under messages_endpoint/providers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): replay a real Claude Code /v1/messages request through the native wire Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): drop legacy covers marker from claude code wire test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): inline the Claude Code request instead of a json fixture Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): drop the legacy covers marker from the new contract Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): add Claude Code /v1/messages customer-journey matrix (native + responses bridge) (#43386) * test(anthropic): add Claude Code /v1/messages customer-journey matrix (native + responses bridge) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): strengthen bot-flagged assertions in the Claude Code matrix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): type the usage mapping parameter in the shared builders Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): drop low-priority Claude Code error and count_tokens tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): send the full 24-tool Claude Code request and pin upstream headers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): drop responses bridge Claude Code tests to keep this PR Anthropic direct only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): fix duplicate WebSearch tool, drop mutation in stream builders, ignore pings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): assert upstream request order in multi-turn Claude Code tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): name Claude Code wire tests by behavior and move provider-agnostic ones to routing and streaming Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): sort imports after moving the Claude Code fixture into _support Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): group anthropic messages tests into feature subfolders Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drop the pre-move anthropic test paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): cover native reasoning translation, response and pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): narrow PR to new reasoning tests, restore moved files and drop non-reasoning tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): scope reasoning tests to reasoning and cover betas, thinking usage and streamed pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): write reasoning cases as literal sent and received fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): require stopped stream blocks and check upstream model on switch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/claude_code.py | 1009 +++++++++++++++++ ...ropic_interleaved_thinking_history_wire.py | 81 ++ ...est_anthropic_model_switch_history_wire.py | 81 ++ ...opic_reasoning_request_translation_wire.py | 482 ++++++++ .../test_anthropic_reasoning_response_wire.py | 63 + ..._anthropic_reasoning_token_pricing_wire.py | 64 ++ 6 files changed, 1780 insertions(+) create mode 100644 tests/integration/_support/claude_code.py create mode 100644 tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_interleaved_thinking_history_wire.py create mode 100644 tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_model_switch_history_wire.py create mode 100644 tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_request_translation_wire.py create mode 100644 tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_response_wire.py create mode 100644 tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_token_pricing_wire.py diff --git a/tests/integration/_support/claude_code.py b/tests/integration/_support/claude_code.py new file mode 100644 index 00000000000..7bb05ce941e --- /dev/null +++ b/tests/integration/_support/claude_code.py @@ -0,0 +1,1009 @@ +"""Shared Claude Code-shaped request builders and upstream stream fixtures for integration contracts.""" + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from functools import reduce +from itertools import chain +from typing import Final + +from integration._support.wire import Request +from pydantic import JsonValue, TypeAdapter + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +ANTHROPIC_API_KEY: Final = "synthetic-anthropic-key" +FABLE: Final = "claude-fable-5-1" +OPUS: Final = "claude-opus-5-5" +CLI_BETA: Final = ( + "claude-code-20250219,interleaved-thinking-2025-05-14,thinking-token-count-2026-05-13," + "context-management-2025-06-27,prompt-caching-scope-2026-01-05" +) +FRONTIER_CLI_BETA: Final = ( + f"{CLI_BETA},mid-conversation-system-2026-04-07,per-turn-control-2026-07-01," + "mid-conversation-tool-changes-2026-07-01,effort-2025-11-24" +) +CACHE: Final = {"type": "ephemeral"} +CONTEXT_MANAGEMENT: Final = {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]} +THINKING_BUDGET: Final = {"budget_tokens": 31999, "type": "enabled", "display": "omitted"} +THINKING_ADAPTIVE: Final = {"type": "adaptive", "display": "omitted"} +CLAUDE_CODE_REASONING_BETAS: Final = ( + "effort-2025-11-24", + "interleaved-thinking-2025-05-14", + "thinking-token-count-2026-05-13", +) +REASONING_FIELDS: Final = ("thinking", "output_config", "reasoning_effort", "temperature") +_OUTPUT_USAGE_KEYS: Final = frozenset({"output_tokens", "output_tokens_details"}) +METADATA_USER_ID: Final = json.dumps( + { + "device_id": "0" * 64, + "account_uuid": "", + "session_id": "00000000-0000-4000-8000-000000000000", + } +) + + +def schema(properties: JsonValue, required: tuple[str, ...]) -> dict[str, JsonValue]: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": properties, + "required": list(required), + "additionalProperties": False, + } + + +def field(description: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"description": description, **extra} + + +def tools() -> tuple[dict[str, JsonValue], ...]: + _MAX: Final = 9007199254740991 + return ( + { + "name": "Agent", + "description": "Launch a new agent to handle complex, multi-step tasks.", + "input_schema": schema( + { + "description": field("A short (3-5 word) description of the task", type="string"), + "prompt": field("The task for the agent to perform", type="string"), + "subagent_type": field("The type of specialized agent to use for this task", type="string"), + "model": field( + "Optional model override for this agent.", + type="string", + enum=["sonnet", "opus", "haiku", "fable"], + ), + "run_in_background": field( + "Agents run in the background by default; you will be notified when one completes.", + type="boolean", + ), + "isolation": field("Isolation mode.", type="string", enum=["worktree", "remote"]), + }, + ("description", "prompt"), + ), + }, + { + "name": "Bash", + "description": "Executes a given bash command and returns its output.", + "input_schema": schema( + { + "command": field("The command to execute", type="string"), + "timeout": field("Optional timeout in milliseconds (max 600000)", type="number"), + "description": field( + "Clear, concise description of what this command does in active voice.", + type="string", + ), + "run_in_background": field("Set to true to run this command in the background.", type="boolean"), + "dangerouslyDisableSandbox": field( + "Set this to true to dangerously override sandbox mode and run commands without sandboxing.", + type="boolean", + ), + }, + ("command",), + ), + }, + { + "name": "CronCreate", + "description": "Schedule a prompt to be enqueued at a future time.", + "input_schema": schema( + { + "cron": field( + 'Standard 5-field cron expression in local time: "M H DoM Mon DoW" (e.g.', + type="string", + ), + "prompt": field("The prompt to enqueue at each fire time.", type="string"), + "recurring": field( + "true (default) = fire on every cron match until deleted or auto-expired after 7 days.", + type="boolean", + ), + "durable": field( + "true = persist to .claude/scheduled_tasks.json and survive restarts.", + type="boolean", + ), + }, + ("cron", "prompt"), + ), + }, + { + "name": "CronDelete", + "description": "Cancel a cron job previously scheduled with CronCreate.", + "input_schema": schema( + { + "id": field("Job ID returned by CronCreate.", type="string"), + }, + ("id",), + ), + }, + { + "name": "CronList", + "description": "List all cron jobs scheduled via CronCreate, both durable (.claude/scheduled_tasks.json) and session-only.", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + }, + { + "name": "Edit", + "description": "Performs exact string replacements in files.", + "input_schema": schema( + { + "file_path": field("The absolute path to the file to modify", type="string"), + "old_string": field("The text to replace", type="string"), + "new_string": field( + "The text to replace it with (must be different from old_string)", type="string" + ), + "replace_all": field( + "Replace all occurrences of old_string (default false)", + default=False, + type="boolean", + ), + }, + ("file_path", "old_string", "new_string"), + ), + }, + { + "name": "EnterWorktree", + "description": "Use this tool ONLY when explicitly instructed to work in a worktree — either by the user directly, or by project instruc", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "name": field("Optional name for a new worktree.", type="string"), + "path": field( + "Path to an existing worktree to switch into instead of creating a new one.", + type="string", + ), + }, + "additionalProperties": False, + }, + }, + { + "name": "ExitWorktree", + "description": "Exit a worktree session created by EnterWorktree and return the session to the original working directory.", + "input_schema": schema( + { + "action": field( + '"keep" leaves the worktree and branch on disk; "remove" deletes both.', + type="string", + enum=["keep", "remove"], + ), + "discard_changes": field( + 'Required true when action is "remove" and the worktree has uncommitted files or unmerged commits.', + type="boolean", + ), + }, + ("action",), + ), + }, + { + "name": "ListAgents", + "description": "Lists agents you can SendMessage to — in-process subagents you spawned, the teammates on your team, other local Claude s", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "channel": field("Not available in this build; leave unset.", type="string", maxLength=256), + "q": field("Not available in this build; leave unset.", type="string", maxLength=256), + }, + "additionalProperties": False, + }, + }, + { + "name": "NotebookEdit", + "description": "Replaces, inserts, or deletes a single cell in a Jupyter notebook (.ipynb file).", + "input_schema": schema( + { + "notebook_path": field( + "The absolute path to the Jupyter notebook file to edit (must be absolute, not relative)", + type="string", + ), + "cell_id": field("The ID of the cell to edit.", type="string"), + "new_source": field("The new source for the cell", type="string"), + "cell_type": field( + "The type of the cell (code or markdown).", + type="string", + enum=["code", "markdown"], + ), + "edit_mode": field( + "The type of edit to make (replace, insert, delete).", + type="string", + enum=["replace", "insert", "delete"], + ), + }, + ("notebook_path", "new_source"), + ), + }, + { + "name": "Read", + "description": "Reads a file from the local filesystem.", + "input_schema": schema( + { + "file_path": field("The absolute path to the file to read", type="string"), + "offset": field("The line number to start reading from.", type="integer", minimum=0, maximum=_MAX), + "limit": field( + "The number of lines to read.", + type="integer", + exclusiveMinimum=0, + maximum=_MAX, + ), + "pages": field('Page range for PDF files (e.g., "1-5", "3", "10-20").', type="string"), + }, + ("file_path",), + ), + }, + { + "name": "ReportFindings", + "description": "Report code-review findings as a typed list so the host UI can render them.", + "input_schema": schema( + { + "level": field( + "Effort level the review ran at", + type="string", + enum=["low", "medium", "high", "xhigh", "max"], + ), + "findings": field( + "Verified findings, most-severe first; empty if none survived", + maxItems=32, + type="array", + items={ + "type": "object", + "properties": { + "file": field("Repo-relative path of the file the finding is in", type="string"), + "line": field( + "1-indexed line the finding anchors to", + type="integer", + minimum=-_MAX, + maximum=_MAX, + ), + "summary": field("One-sentence statement of the defect", type="string"), + "short_summary": field( + "Compressed label for compact UI (≤60 chars): the claim alone, no rationale or consequence clause", + type="string", + maxLength=60, + ), + "failure_scenario": field("Concrete inputs/state → wrong output/crash", type="string"), + "category": field( + "Short kebab-case slug of the finding type, e.g.", + type="string", + maxLength=40, + ), + "verdict": field( + "Set when a verify pass ran; absent on inline-only reviews", + type="string", + enum=["CONFIRMED", "PLAUSIBLE"], + ), + "outcome": field( + "Set ONLY when re-reporting after applying fixes: what happened to this finding", + type="string", + enum=["fixed", "skipped", "no_change_needed"], + ), + }, + "required": ["file", "summary", "failure_scenario"], + "additionalProperties": False, + }, + ), + }, + ("findings",), + ), + }, + { + "name": "ScheduleWakeup", + "description": "Schedule when to resume work in /loop dynamic mode — the user invoked /loop without an interval, asking you to self-pace", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "delaySeconds": field("Seconds from now to wake up.", type="number"), + "reason": field("One short sentence explaining the chosen delay.", type="string"), + "prompt": field("The /loop input to fire on wake-up.", type="string"), + "stop": field( + "Set to true to end the dynamic loop immediately instead of scheduling another wakeup.", + type="boolean", + ), + "noop": field( + "true = nothing changed (you checked and there is nothing to report).", + type="boolean", + ), + }, + "additionalProperties": False, + }, + }, + { + "name": "SendMessage", + "description": "# SendMessage\n\nSend a message to another agent.", + "input_schema": schema( + { + "to": field( + 'Recipient: a name from ListAgents (append its " [ref]" only when a listing or an error shows one), a teammate name, "mai', + type="string", + allOf=[{"pattern": "^[^\\n\\r]*$"}, {"pattern": "^[\\s\\S]{0,300}$"}], + ), + "summary": field( + "A 5-10 word label for your own transcript row (not transmitted — the recipient previews the first line of `message`).", + type="string", + maxLength=200, + ), + "message": field("Plain text message content.", default="", type="string"), + "notify_when_idle": field( + "Ask a session ON THIS MACHINE to send you ONE notice when it next goes idle (finishes its turn with nothing queued) or e", + type="boolean", + ), + }, + ("to", "message"), + ), + }, + { + "name": "Skill", + "description": "Invoke a skill.", + "input_schema": schema( + { + "skill": field("The name of a skill from the available-skills list.", type="string"), + "args": field("Optional arguments for the skill", type="string"), + }, + ("skill",), + ), + }, + { + "name": "TaskCreate", + "description": "Use this tool to create a structured task list for your current coding session.", + "input_schema": schema( + { + "subject": field("A brief title for the task", type="string"), + "description": field("What needs to be done", type="string"), + "activeForm": field( + 'Present continuous form shown in spinner when in_progress (e.g., "Running tests")', + type="string", + ), + "metadata": field( + "Arbitrary metadata to attach to the task", + type="object", + propertyNames={"type": "string"}, + additionalProperties={}, + ), + }, + ("subject", "description"), + ), + }, + { + "name": "TaskGet", + "description": "Use this tool to retrieve a task by its ID from the task list.", + "input_schema": schema( + { + "taskId": field("The ID of the task to retrieve", type="string"), + }, + ("taskId",), + ), + }, + { + "name": "TaskList", + "description": "Use this tool to list all tasks in the task list.", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + }, + { + "name": "TaskStop", + "description": "- Stops a running background task by its ID\n- Takes a task_id parameter identifying the task to stop\n- To stop an agent-", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "task_id": field("The ID of the background task to stop.", type="string"), + "shell_id": field("Deprecated: use task_id instead", type="string"), + }, + "additionalProperties": False, + }, + }, + { + "name": "TaskUpdate", + "description": "Use this tool to update a task in the task list.", + "input_schema": schema( + { + "taskId": field("The ID of the task to update", type="string"), + "subject": field("New subject for the task", type="string"), + "description": field("New description for the task", type="string"), + "activeForm": field( + 'Present continuous form shown in spinner when in_progress (e.g., "Running tests")', + type="string", + ), + "status": field( + "New status for the task", + anyOf=[ + {"type": "string", "enum": ["pending", "in_progress", "completed"]}, + {"type": "string", "const": "deleted"}, + ], + ), + "addBlocks": field("Task IDs that this task blocks", type="array", items={"type": "string"}), + "addBlockedBy": field("Task IDs that block this task", type="array", items={"type": "string"}), + "owner": field("New owner for the task", type="string"), + "metadata": field( + "Metadata keys to merge into the task.", + type="object", + propertyNames={"type": "string"}, + additionalProperties={}, + ), + }, + ("taskId",), + ), + }, + { + "name": "WebFetch", + "description": "IMPORTANT: WebFetch WILL FAIL for authenticated or private URLs.", + "input_schema": schema( + { + "url": field("The URL to fetch content from", type="string", format="uri"), + "prompt": field("The prompt to run on the fetched content", type="string"), + }, + ("url", "prompt"), + ), + }, + { + "name": "WebSearch", + "description": "- Allows Claude to search the web and use the results to inform responses\n- Provides up-to-date information for current ", + "input_schema": schema( + { + "query": field("The search query to use", type="string", minLength=2), + "allowed_domains": field( + "Only include search results from these domains", type="array", items={"type": "string"} + ), + "blocked_domains": field( + "Never include search results from these domains", type="array", items={"type": "string"} + ), + }, + ("query",), + ), + }, + { + "name": "Workflow", + "description": "Execute a workflow script that orchestrates multiple subagents deterministically.", + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "script": field("Self-contained workflow script.", type="string", maxLength=524288), + "name": field( + "Name of a predefined workflow (built-in or from .claude/workflows/).", type="string" + ), + "description": field( + "Ignored — set the workflow description in the script's `meta` block.", type="string" + ), + "title": field("Ignored — set the workflow title in the script's `meta` block.", type="string"), + "args": field("Optional input value exposed to the script as the global `args`, verbatim."), + "scriptPath": field("Path to a workflow script file on disk.", type="string"), + "resumeFromRunId": field( + "Run ID of a prior Workflow invocation to resume from.", + type="string", + pattern="^wf_[a-z0-9-]{6,}$", + ), + }, + "additionalProperties": False, + }, + }, + { + "name": "Write", + "description": "Writes a file to the local filesystem.", + "input_schema": schema( + { + "file_path": field( + "The absolute path to the file to write (must be absolute, not relative)", type="string" + ), + "content": field("The content to write to the file", type="string"), + }, + ("file_path", "content"), + ), + }, + ) + + +def system_blocks() -> tuple[dict[str, JsonValue], ...]: + return ( + {"type": "text", "text": "x-anthropic-billing-header: cc_version=2.1.283.00; cc_entrypoint=sdk-cli;"}, + {"type": "text", "text": "Synthetic agent identity system prompt.", "cache_control": CACHE}, + {"type": "text", "text": "Synthetic interactive agent instructions.", "cache_control": CACHE}, + ) + + +def claude_code_request(cache_bust: str) -> dict[str, JsonValue]: + reminders: Final = ( + f"\n{cache_bust}\n", + "\nSynthetic model identity reminder.\n", + "\nSynthetic agent types reminder.\n", + "\nSynthetic skills reminder.\n", + "\n15000000 tokens left\n", + "\nSynthetic date reminder.\n", + "\nSynthetic attribution reminder.\n", + ) + return { + "model": "", + "system": list(system_blocks()), + "messages": [ + { + "role": "user", + "content": [ + *[{"type": "text", "text": reminder} for reminder in reminders], + {"type": "text", "text": "Reply with exactly the word PONG", "cache_control": CACHE}, + ], + } + ], + "tools": list(tools()), + "metadata": {"user_id": METADATA_USER_ID}, + "max_tokens": 32000, + "thinking": dict(THINKING_BUDGET), + "context_management": dict(CONTEXT_MANAGEMENT), + "stream": True, + } + + +def frontier_request( + cache_bust: str, + effort: str, + max_tokens: int, + prompt_text: str = "Reply with exactly the word PONG", + stream: bool = True, +) -> dict[str, JsonValue]: + return { + "model": "", + "system": list(system_blocks()), + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"\n{cache_bust}\n"}, + {"type": "text", "text": prompt_text}, + ], + }, + { + "role": "system", + "content": [ + { + "type": "text", + "text": "# Environment\nSynthetic environment block.", + "cache_control": CACHE, + } + ], + }, + ], + "tools": list(tools()), + "metadata": {"user_id": METADATA_USER_ID}, + "max_tokens": max_tokens, + "thinking": dict(THINKING_ADAPTIVE), + "context_management": dict(CONTEXT_MANAGEMENT), + "output_config": {"effort": effort}, + "stream": stream, + } + + +def tool_loop_turn2( + base: dict[str, JsonValue], + assistant_content: tuple[dict[str, JsonValue], ...], + tool_results: tuple[tuple[str, JsonValue], ...], +) -> dict[str, JsonValue]: + return { + **base, + "messages": [ + *base["messages"], + {"role": "assistant", "content": list(assistant_content)}, + { + "role": "user", + "content": [ + {"tool_use_id": tool_use_id, "type": "tool_result", "content": content} + for tool_use_id, content in tool_results + ], + }, + { + "role": "system", + "content": [ + { + "type": "text", + "text": "14999970 tokens left", + "cache_control": CACHE, + }, + { + "type": "text", + "text": "First privately list what you need next; then request every item that doesn't depend on another's result in this one response.", + }, + ], + }, + ], + } + + +def cli_headers(key: str, beta: str = CLI_BETA) -> dict[str, str]: + return { + "accept": "application/json", + "content-type": "application/json", + "user-agent": "claude-cli/2.1.283 (external, sdk-cli)", + "x-claude-code-session-id": "00000000-0000-4000-8000-000000000000", + "x-stainless-arch": "x64", + "x-stainless-lang": "js", + "x-stainless-os": "Linux", + "x-stainless-package-version": "0.112.1", + "x-stainless-retry-count": "0", + "x-stainless-runtime": "node", + "x-stainless-runtime-version": "v26.3.0", + "x-stainless-timeout": "600", + "anthropic-beta": beta, + "anthropic-dangerous-direct-browser-access": "true", + "anthropic-version": "2023-06-01", + "x-app": "cli", + "x-api-key": key, + } + + +def sse_frame(event: str, data: JsonValue) -> bytes: + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + +def sse_events(text: str) -> tuple[tuple[str, dict[str, object]], ...]: + frames: Final = tuple(frame for frame in text.split("\n\n") if frame.strip()) + return tuple( + ( + event, + json.loads(next(line.removeprefix("data: ") for line in frame.splitlines() if line.startswith("data: "))), + ) + for frame in frames + if (event := next(line.removeprefix("event: ") for line in frame.splitlines() if line.startswith("event: "))) + != "ping" + ) + + +def reasoning_betas(anthropic_beta: str) -> tuple[str, ...]: + return tuple(sorted(beta for beta in anthropic_beta.split(",") if beta in CLAUDE_CODE_REASONING_BETAS)) + + +def body_diff(expected: Mapping[str, JsonValue], body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return { + key: {"expected": expected.get(key), "upstream": body.get(key)} + for key in expected.keys() | body.keys() + if expected.get(key) != body.get(key) + } + + +@dataclass(frozen=True, slots=True) +class Forwarded: + model: JsonValue + reasoning: dict[str, JsonValue] + assistant_history: tuple[JsonValue, ...] + other_changes: dict[str, JsonValue] + reasoning_betas: tuple[str, ...] + + +def _is_assistant_turn(message: JsonValue) -> bool: + return isinstance(message, dict) and message.get("role") == "assistant" + + +def _assistant_history(body: Mapping[str, JsonValue]) -> tuple[JsonValue, ...]: + messages: Final = body.get("messages") + if not isinstance(messages, list): + return () + return tuple( + message["content"] for message in messages if isinstance(message, dict) and _is_assistant_turn(message) + ) + + +def _without_assistant_turns(messages: JsonValue) -> JsonValue: + if not isinstance(messages, list): + return messages + return [message for message in messages if not _is_assistant_turn(message)] + + +def _unrelated_fields(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + excluded: Final = frozenset({*REASONING_FIELDS, "model"}) + return { + key: _without_assistant_turns(value) if key == "messages" else value + for key, value in body.items() + if key not in excluded + } + + +def forwarded(sent: Mapping[str, JsonValue], request: Request) -> Forwarded: + body: Final = JSON_OBJECT.validate_json(request.body) + return Forwarded( + model=body.get("model"), + reasoning={field: body[field] for field in REASONING_FIELDS if field in body}, + assistant_history=_assistant_history(body), + other_changes=body_diff(_unrelated_fields(sent), _unrelated_fields(body)), + reasoning_betas=reasoning_betas(request.headers.get("anthropic-beta", "")), + ) + + +def _appended(block: Mapping[str, JsonValue], key: str, text: str) -> dict[str, JsonValue]: + return {**block, key: f"{block.get(key) or ''}{text}"} + + +def _with_delta(block: dict[str, JsonValue], delta: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + match delta: + case {"type": "thinking_delta", "thinking": str(text)}: + return _appended(block, "thinking", text) + case {"type": "signature_delta", "signature": str(text)}: + return _appended(block, "signature", text) + case {"type": "text_delta", "text": str(text)}: + return _appended(block, "text", text) + case {"type": "input_json_delta", "partial_json": str(text)}: + return _appended(block, "partial_json", text) + case _: + return block + + +def _with_event( + blocks: tuple[dict[str, JsonValue], ...], event: tuple[str, dict[str, JsonValue]] +) -> tuple[dict[str, JsonValue], ...]: + match event: + case ("content_block_start", {"content_block": dict() as block}): + return (*blocks, JSON_OBJECT.validate_python(block)) + case ("content_block_delta", {"index": int(index), "delta": dict() as delta}): + return ( + *blocks[:index], + _with_delta(blocks[index], JSON_OBJECT.validate_python(delta)), + *blocks[index + 1 :], + ) + case _: + return blocks + + +def _finished(block: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + partial_json: Final = block.get("partial_json") + if not isinstance(partial_json, str): + return dict(block) + return {**{key: value for key, value in block.items() if key != "partial_json"}, "input": json.loads(partial_json)} + + +def _client_events(stream: str) -> tuple[tuple[str, dict[str, JsonValue]], ...]: + return tuple((event, JSON_OBJECT.validate_python(data)) for event, data in sse_events(stream)) + + +def _stopped_indices(events: tuple[tuple[str, dict[str, JsonValue]], ...]) -> frozenset[JsonValue]: + return frozenset(data.get("index") for event, data in events if event == "content_block_stop") + + +def streamed_content(stream: str) -> list[dict[str, JsonValue]]: + events: Final = _client_events(stream) + stopped: Final = _stopped_indices(events) + blocks: Final = reduce(_with_event, events, ()) + return [_finished(block) for index, block in enumerate(blocks) if index in stopped] + + +def streamed_usage(stream: str) -> JsonValue: + return next(data.get("usage") for event, data in reversed(_client_events(stream)) if event == "message_delta") + + +def _start_usage(usage: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {key: value for key, value in usage.items() if key not in _OUTPUT_USAGE_KEYS} + + +def _delta_usage(usage: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {key: value for key, value in usage.items() if key in _OUTPUT_USAGE_KEYS} + + +def _block_start(index: int, block: Mapping[str, JsonValue]) -> bytes: + return sse_frame("content_block_start", {"type": "content_block_start", "index": index, "content_block": block}) + + +def _block_delta(index: int, delta: Mapping[str, JsonValue]) -> bytes: + return sse_frame("content_block_delta", {"type": "content_block_delta", "index": index, "delta": delta}) + + +def _block_stop(index: int) -> bytes: + return sse_frame("content_block_stop", {"type": "content_block_stop", "index": index}) + + +def _block_frames(index: int, block: Mapping[str, JsonValue]) -> tuple[bytes, ...]: + match block.get("type"): + case "thinking": + return ( + _block_start(index, {"type": "thinking", "thinking": "", "signature": ""}), + _block_delta(index, {"type": "thinking_delta", "thinking": block["thinking"]}), + _block_delta(index, {"type": "signature_delta", "signature": block["signature"]}), + _block_stop(index), + ) + case "text": + return ( + _block_start(index, {"type": "text", "text": ""}), + _block_delta(index, {"type": "text_delta", "text": block["text"]}), + _block_stop(index), + ) + case _: + return (_block_start(index, block), _block_stop(index)) + + +def message_reply( + identity: str, model: str, content: tuple[dict[str, JsonValue], ...], usage: dict[str, JsonValue] +) -> bytes: + return json.dumps( + { + "id": identity, + "type": "message", + "role": "assistant", + "model": model, + "content": list(content), + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": usage, + } + ).encode() + + +def message_stream( + identity: str, model: str, content: tuple[dict[str, JsonValue], ...], usage: dict[str, JsonValue] +) -> tuple[bytes, ...]: + start: Final = sse_frame( + "message_start", + { + "type": "message_start", + "message": { + "id": identity, + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": _start_usage(usage), + }, + }, + ) + delta: Final = sse_frame( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": _delta_usage(usage), + }, + ) + blocks: Final = chain.from_iterable(_block_frames(index, block) for index, block in enumerate(content)) + return (start, *blocks, delta, sse_frame("message_stop", {"type": "message_stop"})) + + +def text_stream(identity: str, model: str, text: str, usage: dict[str, int]) -> tuple[bytes, ...]: + return ( + sse_frame( + "message_start", + { + "type": "message_start", + "message": { + "id": identity, + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": _start_usage(usage), + }, + }, + ), + sse_frame( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + sse_frame( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}, + ), + sse_frame("content_block_stop", {"type": "content_block_stop", "index": 0}), + sse_frame( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": usage["output_tokens"]}, + }, + ), + sse_frame("message_stop", {"type": "message_stop"}), + ) + + +def _tool_use_frames(index: int, tool_id: str, name: str, tool_input: JsonValue) -> tuple[bytes, ...]: + arguments: Final = json.dumps(tool_input) + return ( + sse_frame( + "content_block_start", + { + "type": "content_block_start", + "index": index, + "content_block": {"type": "tool_use", "id": tool_id, "name": name, "input": {}}, + }, + ), + sse_frame( + "content_block_delta", + { + "type": "content_block_delta", + "index": index, + "delta": {"type": "input_json_delta", "partial_json": arguments[: len(arguments) // 2]}, + }, + ), + sse_frame( + "content_block_delta", + { + "type": "content_block_delta", + "index": index, + "delta": {"type": "input_json_delta", "partial_json": arguments[len(arguments) // 2 :]}, + }, + ), + sse_frame("content_block_stop", {"type": "content_block_stop", "index": index}), + ) + + +def tool_use_stream( + identity: str, + model: str, + thinking: str, + signature: str, + tool_calls: tuple[tuple[str, str, JsonValue], ...], + usage: dict[str, int], +) -> tuple[bytes, ...]: + head: Final = ( + sse_frame( + "message_start", + { + "type": "message_start", + "message": { + "id": identity, + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": _start_usage(usage), + }, + }, + ), + sse_frame( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "thinking", "thinking": ""}}, + ), + sse_frame( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": thinking}}, + ), + sse_frame( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "signature_delta", "signature": signature}}, + ), + sse_frame("content_block_stop", {"type": "content_block_stop", "index": 0}), + ) + tail: Final = ( + sse_frame( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": usage["output_tokens"]}, + }, + ), + sse_frame("message_stop", {"type": "message_stop"}), + ) + frames: Final = ( + *head, + *chain.from_iterable( + _tool_use_frames(index, tool_id, name, tool_input) + for index, (tool_id, name, tool_input) in enumerate(tool_calls, start=1) + ), + *tail, + ) + return frames diff --git a/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_interleaved_thinking_history_wire.py b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_interleaved_thinking_history_wire.py new file mode 100644 index 00000000000..ac2247ff09a --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_interleaved_thinking_history_wire.py @@ -0,0 +1,81 @@ +import uuid +from typing import Final + +from integration._support import claude_code as cc +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +_READ_A: Final = {"type": "tool_use", "id": "toolu_a", "name": "Read", "input": {"file_path": "/tmp/cc_probe/a.txt"}} +_READ_B: Final = {"type": "tool_use", "id": "toolu_b", "name": "Read", "input": {"file_path": "/tmp/cc_probe/b.txt"}} + + +def test_interleaved_thinking_history_reaches_anthropic_and_interleaved_blocks_stream_back(gateway: Gateway) -> None: + turn1: Final = cc.frontier_request( + f"cache-bust-{uuid.uuid4().hex}", + "high", + 64000, + prompt_text="Read /tmp/cc_probe/a.txt then /tmp/cc_probe/b.txt one at a time and reply with both words", + ) + turn2: Final = cc.tool_loop_turn2( + turn1, ({"type": "thinking", "thinking": "plan", "signature": "sig1"}, _READ_A), (("toolu_a", "ALPHA"),) + ) + turn3: Final = cc.tool_loop_turn2( + turn2, + ({"type": "thinking", "thinking": "got A", "signature": "sig2"}, {"type": "text", "text": "got A"}, _READ_B), + (("toolu_b", "BRAVO"),), + ) + + def respond(request: Request) -> Reply: + if b"toolu_b" not in request.body: + return Reply( + content_type="text/event-stream", + chunks=cc.text_stream("msg_il_turn2", cc.FABLE, "got A", {"input_tokens": 20, "output_tokens": 4}), + ) + return Reply( + content_type="text/event-stream", + chunks=cc.message_stream( + f"msg_il_{uuid.uuid4().hex}", + cc.FABLE, + ( + {"type": "thinking", "thinking": "got B", "signature": "sig3"}, + {"type": "text", "text": "got B"}, + { + "type": "tool_use", + "id": "toolu_c", + "name": "Read", + "input": {"file_path": "/tmp/cc_probe/c.txt"}, + }, + ), + {"input_tokens": 20, "output_tokens": 12}, + ), + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{cc.FABLE}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY) + headers: Final = cc.cli_headers(gateway.key, cc.FRONTIER_CLI_BETA) + response2: Final = gateway.request( + "POST", "/v1/messages", {**turn2, "model": model}, params={"beta": "true"}, headers=headers + ) + assert response2.status_code == 200, response2.text + response3: Final = gateway.request( + "POST", "/v1/messages", {**turn3, "model": model}, params={"beta": "true"}, headers=headers + ) + assert response3.status_code == 200, response3.text + received: Final = wire.drain() + assert len(received) == 2, received + second: Final = cc.forwarded(turn2, received[0]) + third: Final = cc.forwarded(turn3, received[1]) + assert second.assistant_history == ([{"type": "thinking", "thinking": "plan", "signature": "sig1"}, _READ_A],) + assert third.assistant_history == ( + [{"type": "thinking", "thinking": "plan", "signature": "sig1"}, _READ_A], + [{"type": "thinking", "thinking": "got A", "signature": "sig2"}, {"type": "text", "text": "got A"}, _READ_B], + ), third.assistant_history + adaptive_high: Final = {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "high"}} + assert (second.reasoning, third.reasoning) == (adaptive_high, adaptive_high) + assert (second.other_changes, third.other_changes) == ({}, {}) + assert (second.reasoning_betas, third.reasoning_betas) == (cc.CLAUDE_CODE_REASONING_BETAS,) * 2 + assert cc.streamed_content(response3.text) == [ + {"type": "thinking", "thinking": "got B", "signature": "sig3"}, + {"type": "text", "text": "got B"}, + {"type": "tool_use", "id": "toolu_c", "name": "Read", "input": {"file_path": "/tmp/cc_probe/c.txt"}}, + ], response3.text diff --git a/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_model_switch_history_wire.py b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_model_switch_history_wire.py new file mode 100644 index 00000000000..003fe870f16 --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_model_switch_history_wire.py @@ -0,0 +1,81 @@ +import uuid +from typing import Final + +from integration._support import claude_code as cc +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + + +def test_mid_loop_model_switch_replays_thinking_history_and_reasoning_unchanged(gateway: Gateway) -> None: + turn1: Final = cc.frontier_request( + f"cache-bust-{uuid.uuid4().hex}", + "high", + 64000, + prompt_text="Read /tmp/cc_probe/hello.txt and reply with its single word", + ) + turn2: Final = cc.tool_loop_turn2( + turn1, + ( + {"type": "thinking", "thinking": "need to read the file", "signature": "sig_anthropic_1"}, + { + "type": "tool_use", + "id": "toolu_read_1", + "name": "Read", + "input": {"file_path": "/tmp/cc_probe/hello.txt"}, + }, + ), + (("toolu_read_1", "1\tPROBE\n2\t"),), + ) + + def respond(request: Request) -> Reply: + if b"tool_result" not in request.body: + return Reply( + content_type="text/event-stream", + chunks=cc.tool_use_stream( + f"msg_{uuid.uuid4().hex}", + cc.FABLE, + "need to read the file", + "sig_anthropic_1", + (("toolu_read_1", "Read", {"file_path": "/tmp/cc_probe/hello.txt"}),), + {"input_tokens": 20, "output_tokens": 10}, + ), + ) + return Reply( + content_type="text/event-stream", + chunks=cc.text_stream( + f"msg_{uuid.uuid4().hex}", cc.OPUS, "PROBE", {"input_tokens": 30, "output_tokens": 3} + ), + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + fable: Final = scenario.model(model=f"anthropic/{cc.FABLE}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY) + opus: Final = scenario.model(model=f"anthropic/{cc.OPUS}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY) + headers: Final = cc.cli_headers(gateway.key, cc.FRONTIER_CLI_BETA) + response1: Final = gateway.request( + "POST", "/v1/messages", {**turn1, "model": fable}, params={"beta": "true"}, headers=headers + ) + assert response1.status_code == 200, response1.text + response2: Final = gateway.request( + "POST", "/v1/messages", {**turn2, "model": opus}, params={"beta": "true"}, headers=headers + ) + assert response2.status_code == 200, response2.text + received: Final = wire.drain() + assert len(received) == 2, received + to_fable: Final = cc.forwarded(turn1, received[0]) + to_opus: Final = cc.forwarded(turn2, received[1]) + assert (to_fable.model, to_opus.model) == (cc.FABLE, cc.OPUS), received + assert to_opus.assistant_history == ( + [ + {"type": "thinking", "thinking": "need to read the file", "signature": "sig_anthropic_1"}, + { + "type": "tool_use", + "id": "toolu_read_1", + "name": "Read", + "input": {"file_path": "/tmp/cc_probe/hello.txt"}, + }, + ], + ), to_opus.assistant_history + adaptive_high: Final = {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "high"}} + assert (to_fable.reasoning, to_opus.reasoning) == (adaptive_high, adaptive_high) + assert (to_fable.other_changes, to_opus.other_changes) == ({}, {}) + assert (to_fable.reasoning_betas, to_opus.reasoning_betas) == (cc.CLAUDE_CODE_REASONING_BETAS,) * 2 diff --git a/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_request_translation_wire.py b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_request_translation_wire.py new file mode 100644 index 00000000000..4fc99e2dd67 --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_request_translation_wire.py @@ -0,0 +1,482 @@ +import uuid +from collections.abc import Mapping +from typing import Final + +import pytest +from integration._support import claude_code as cc +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + + +def _claude_code_turn(sent: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + default_turn: Final = cc.claude_code_request(f"cache-bust-{uuid.uuid4().hex}") + without_reasoning: Final = {key: value for key, value in default_turn.items() if key != "thinking"} + return {**without_reasoning, "stream": False, **sent} + + +def _forward(gateway: Gateway, upstream_model: str, sent: Mapping[str, JsonValue]) -> cc.Forwarded: + client_body: Final = _claude_code_turn(sent) + + def respond(request: Request) -> Reply: + return Reply( + body=cc.message_reply( + f"msg_{uuid.uuid4().hex}", + upstream_model, + ({"type": "text", "text": "PONG"},), + {"input_tokens": 12, "output_tokens": 4}, + ) + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{upstream_model}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + {**client_body, "model": model}, + headers=cc.cli_headers(gateway.key, cc.FRONTIER_CLI_BETA), + ) + assert response.status_code == 200, response.text + received: Final = wire.drain() + assert len(received) == 1, received + return cc.forwarded(client_body, received[0]) + + +@pytest.mark.parametrize( + ("upstream_model", "sent", "received"), + ( + pytest.param( + "claude-haiku-4-5", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "low"}}, + {"thinking": {"type": "enabled", "budget_tokens": 1024}}, + id="haiku-4.5-low", + ), + pytest.param( + "claude-haiku-4-5", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "medium"}}, + {"thinking": {"type": "enabled", "budget_tokens": 2048}}, + id="haiku-4.5-medium", + ), + pytest.param( + "claude-haiku-4-5", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "high"}}, + {"thinking": {"type": "enabled", "budget_tokens": 4096}}, + id="haiku-4.5-high", + ), + pytest.param( + "claude-haiku-4-5", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "xhigh"}}, + {"thinking": {"type": "enabled", "budget_tokens": 8192}}, + id="haiku-4.5-xhigh", + ), + pytest.param( + "claude-haiku-4-5", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "max"}}, + {"thinking": {"type": "enabled", "budget_tokens": 16384}}, + id="haiku-4.5-max", + ), + pytest.param( + "claude-haiku-4-5", + { + "thinking": {"type": "adaptive", "display": "omitted"}, + "output_config": {"effort": "max"}, + "max_tokens": 4000, + }, + {"thinking": {"type": "enabled", "budget_tokens": 3999}}, + id="haiku-4.5-budget-capped-below-max-tokens", + ), + pytest.param( + "claude-haiku-4-5", + { + "thinking": {"type": "adaptive", "display": "omitted"}, + "output_config": {"effort": "high"}, + "max_tokens": 1024, + }, + {}, + id="haiku-4.5-max-tokens-below-minimum-budget", + ), + pytest.param( + "claude-opus-4-5", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "high"}}, + {"output_config": {"effort": "high"}}, + id="opus-4.5-keeps-effort-drops-adaptive", + ), + pytest.param( + "claude-opus-4-5", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "xhigh"}}, + {"thinking": {"type": "enabled", "budget_tokens": 8192}}, + id="opus-4.5-xhigh-falls-back-to-budget", + ), + pytest.param( + "claude-opus-4-6", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "high"}}, + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "high"}}, + id="opus-4.6-unchanged", + ), + pytest.param( + "claude-opus-4-7", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "xhigh"}}, + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "xhigh"}}, + id="opus-4.7-unchanged", + ), + pytest.param( + "claude-fable-5-1", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "high"}}, + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "high"}}, + id="fable-5.1-unchanged", + ), + pytest.param( + "claude-opus-5-5", + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "xhigh"}}, + {"thinking": {"type": "adaptive", "display": "omitted"}, "output_config": {"effort": "xhigh"}}, + id="opus-5.5-unchanged", + ), + ), +) +def test_adaptive_thinking_and_effort_are_rewritten_only_for_models_without_adaptive_thinking( + gateway: Gateway, upstream_model: str, sent: dict[str, JsonValue], received: dict[str, JsonValue] +) -> None: + forwarded: Final = _forward(gateway, upstream_model, sent) + assert forwarded.reasoning == received, forwarded + assert forwarded.other_changes == {}, forwarded.other_changes + assert forwarded.reasoning_betas == cc.CLAUDE_CODE_REASONING_BETAS, forwarded.reasoning_betas + + +@pytest.mark.parametrize( + ("upstream_model", "sent", "received"), + ( + pytest.param( + "claude-opus-4-7", + {"thinking": {"type": "enabled", "budget_tokens": 1024}}, + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}, + id="opus-4.7-1024-is-low", + ), + pytest.param( + "claude-opus-4-7", + {"thinking": {"type": "enabled", "budget_tokens": 2048}}, + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "medium"}}, + id="opus-4.7-2048-is-medium", + ), + pytest.param( + "claude-opus-4-7", + {"thinking": {"type": "enabled", "budget_tokens": 4096}}, + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}, + id="opus-4.7-4096-is-high", + ), + pytest.param( + "claude-opus-4-7", + {"thinking": {"type": "enabled", "budget_tokens": 8192}}, + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}, + id="opus-4.7-8192-is-xhigh", + ), + pytest.param( + "claude-opus-4-7", + {"thinking": {"type": "enabled", "budget_tokens": 8192}, "output_config": {"effort": "medium"}}, + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "medium"}}, + id="opus-4.7-keeps-the-callers-effort", + ), + pytest.param( + "claude-haiku-4-5", + {"thinking": {"type": "enabled", "budget_tokens": 2048}}, + {"thinking": {"type": "enabled", "budget_tokens": 2048}}, + id="haiku-4.5-unchanged", + ), + ), +) +def test_legacy_thinking_budget_becomes_adaptive_effort_only_on_models_that_reject_budgets( + gateway: Gateway, upstream_model: str, sent: dict[str, JsonValue], received: dict[str, JsonValue] +) -> None: + forwarded: Final = _forward(gateway, upstream_model, sent) + assert forwarded.reasoning == received, forwarded + assert forwarded.other_changes == {}, forwarded.other_changes + assert forwarded.reasoning_betas == cc.CLAUDE_CODE_REASONING_BETAS, forwarded.reasoning_betas + + +@pytest.mark.parametrize( + ("upstream_model", "sent", "received"), + ( + pytest.param( + "claude-fable-5-1", + {"thinking": {"type": "disabled"}}, + {}, + id="fable-5.1-always-thinks-so-disabled-is-dropped", + ), + pytest.param( + "claude-opus-4-7", + {"thinking": {"type": "disabled"}}, + {"thinking": {"type": "disabled"}}, + id="opus-4.7-keeps-disabled", + ), + ), +) +def test_disabled_thinking_is_dropped_only_for_always_on_thinking_models( + gateway: Gateway, upstream_model: str, sent: dict[str, JsonValue], received: dict[str, JsonValue] +) -> None: + forwarded: Final = _forward(gateway, upstream_model, sent) + assert forwarded.reasoning == received, forwarded + assert forwarded.other_changes == {}, forwarded.other_changes + assert forwarded.reasoning_betas == cc.CLAUDE_CODE_REASONING_BETAS, forwarded.reasoning_betas + + +@pytest.mark.parametrize( + ("upstream_model", "sent", "received"), + ( + pytest.param( + "claude-opus-4-7", + {"reasoning_effort": "minimal"}, + {"thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "low"}}, + id="opus-4.7-minimal", + ), + pytest.param( + "claude-opus-4-7", + {"reasoning_effort": "low"}, + {"thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "low"}}, + id="opus-4.7-low", + ), + pytest.param( + "claude-opus-4-7", + {"reasoning_effort": "medium"}, + {"thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "medium"}}, + id="opus-4.7-medium", + ), + pytest.param( + "claude-opus-4-7", + {"reasoning_effort": "high"}, + {"thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "high"}}, + id="opus-4.7-high", + ), + pytest.param( + "claude-opus-4-7", + {"reasoning_effort": "xhigh"}, + {"thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "xhigh"}}, + id="opus-4.7-xhigh", + ), + pytest.param( + "claude-opus-4-7", + {"reasoning_effort": "max"}, + {"thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "max"}}, + id="opus-4.7-max", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "minimal"}, + {"thinking": {"type": "enabled", "budget_tokens": 1024}}, + id="haiku-4.5-minimal", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "low"}, + {"thinking": {"type": "enabled", "budget_tokens": 1024}}, + id="haiku-4.5-low", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "medium"}, + {"thinking": {"type": "enabled", "budget_tokens": 2048}}, + id="haiku-4.5-medium", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "high"}, + {"thinking": {"type": "enabled", "budget_tokens": 4096}}, + id="haiku-4.5-high", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "xhigh"}, + {"thinking": {"type": "enabled", "budget_tokens": 8192}}, + id="haiku-4.5-xhigh", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "max"}, + {"thinking": {"type": "enabled", "budget_tokens": 16384}}, + id="haiku-4.5-max", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "max", "max_tokens": 4000}, + {"thinking": {"type": "enabled", "budget_tokens": 3999}}, + id="haiku-4.5-budget-capped-below-max-tokens", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "high", "max_tokens": 1024}, + {}, + id="haiku-4.5-max-tokens-below-minimum-budget", + ), + pytest.param( + "claude-opus-4-7", + { + "reasoning_effort": "none", + "thinking": {"type": "adaptive", "display": "omitted"}, + "output_config": {"effort": "high"}, + }, + {}, + id="none-clears-thinking-and-effort", + ), + pytest.param( + "claude-haiku-4-5", + {"reasoning_effort": "high", "thinking": {"type": "enabled", "budget_tokens": 2000}}, + {"thinking": {"type": "enabled", "budget_tokens": 2000}}, + id="callers-thinking-wins", + ), + pytest.param( + "claude-opus-4-7", + {"reasoning_effort": "high", "output_config": {"effort": "low"}}, + {"thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "low"}}, + id="callers-effort-wins", + ), + ), +) +def test_reasoning_effort_becomes_the_thinking_shape_each_model_accepts( + gateway: Gateway, upstream_model: str, sent: dict[str, JsonValue], received: dict[str, JsonValue] +) -> None: + forwarded: Final = _forward(gateway, upstream_model, sent) + assert forwarded.reasoning == received, forwarded + assert forwarded.other_changes == {}, forwarded.other_changes + assert forwarded.reasoning_betas == cc.CLAUDE_CODE_REASONING_BETAS, forwarded.reasoning_betas + + +@pytest.mark.parametrize( + ("upstream_model", "sent", "received"), + ( + pytest.param( + "claude-haiku-4-5", + { + "temperature": 0, + "thinking": {"type": "adaptive", "display": "omitted"}, + "output_config": {"effort": "high"}, + }, + {"thinking": {"type": "enabled", "budget_tokens": 4096}}, + id="haiku-4.5-drops-temperature-0-with-effort", + ), + pytest.param( + "claude-opus-4-5", + { + "temperature": 0, + "thinking": {"type": "adaptive", "display": "omitted"}, + "output_config": {"effort": "high"}, + }, + {"output_config": {"effort": "high"}}, + id="opus-4.5-drops-temperature-0-with-effort", + ), + pytest.param( + "claude-haiku-4-5", + {"temperature": 0, "thinking": {"type": "enabled", "budget_tokens": 2048}}, + {"thinking": {"type": "enabled", "budget_tokens": 2048}}, + id="haiku-4.5-drops-temperature-0-with-budget", + ), + pytest.param( + "claude-haiku-4-5", + {"temperature": 1, "thinking": {"type": "enabled", "budget_tokens": 2048}}, + {"temperature": 1, "thinking": {"type": "enabled", "budget_tokens": 2048}}, + id="haiku-4.5-keeps-temperature-1", + ), + pytest.param( + "claude-haiku-4-5", + {"temperature": 0}, + {"temperature": 0}, + id="haiku-4.5-keeps-temperature-without-thinking", + ), + pytest.param( + "claude-opus-4-6", + { + "temperature": 0, + "thinking": {"type": "adaptive", "display": "omitted"}, + "output_config": {"effort": "high"}, + }, + { + "temperature": 0, + "thinking": {"type": "adaptive", "display": "omitted"}, + "output_config": {"effort": "high"}, + }, + id="opus-4.6-adaptive-keeps-temperature", + ), + ), +) +def test_temperature_is_dropped_only_when_a_non_adaptive_model_thinks( + gateway: Gateway, upstream_model: str, sent: dict[str, JsonValue], received: dict[str, JsonValue] +) -> None: + forwarded: Final = _forward(gateway, upstream_model, sent) + assert forwarded.reasoning == received, forwarded + assert forwarded.other_changes == {}, forwarded.other_changes + assert forwarded.reasoning_betas == cc.CLAUDE_CODE_REASONING_BETAS, forwarded.reasoning_betas + + +def _tool_loop(assistant_content: list[JsonValue]) -> dict[str, JsonValue]: + first_turn: Final = cc.claude_code_request(f"cache-bust-{uuid.uuid4().hex}")["messages"] + assert isinstance(first_turn, list) + tool_result: Final = {"type": "tool_result", "tool_use_id": "toolu_01", "content": "ok"} + return { + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "messages": [ + *first_turn, + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": [tool_result]}, + ], + } + + +@pytest.mark.parametrize( + ("sent_history", "received_history"), + ( + pytest.param( + [ + {"type": "thinking", "thinking": "bridge reasoning", "signature": "litellm_encrypted_reasoning:gAAAAB"}, + {"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAAC"}, + {"type": "thinking", "thinking": "check the config", "signature": "EqQBCkgIBRABGAIiQL"}, + {"type": "tool_use", "id": "toolu_01", "name": "Read", "input": {"file_path": "/repo/config.yaml"}}, + ], + [ + {"type": "thinking", "thinking": "check the config", "signature": "EqQBCkgIBRABGAIiQL"}, + {"type": "tool_use", "id": "toolu_01", "name": "Read", "input": {"file_path": "/repo/config.yaml"}}, + ], + id="encrypted-reasoning-from-another-provider-stripped-anthropic-signed-kept", + ), + pytest.param( + [ + {"type": "thinking", "thinking": "", "signature": "EqQBCkgIBRABGAIiQM"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzix"}, + {"type": "tool_use", "id": "toolu_01", "name": "Read", "input": {"file_path": "/repo/config.yaml"}}, + ], + [ + {"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzix"}, + {"type": "tool_use", "id": "toolu_01", "name": "Read", "input": {"file_path": "/repo/config.yaml"}}, + ], + id="empty-thinking-stripped-redacted-thinking-kept", + ), + ), +) +def test_thinking_history_keeps_only_blocks_anthropic_can_verify( + gateway: Gateway, sent_history: list[JsonValue], received_history: list[JsonValue] +) -> None: + forwarded: Final = _forward(gateway, "claude-haiku-4-5", _tool_loop(sent_history)) + assert forwarded.assistant_history == (received_history,), forwarded.assistant_history + assert forwarded.reasoning == {"thinking": {"type": "enabled", "budget_tokens": 2048}}, forwarded + assert forwarded.other_changes == {}, forwarded.other_changes + assert forwarded.reasoning_betas == cc.CLAUDE_CODE_REASONING_BETAS, forwarded.reasoning_betas + + +@pytest.mark.parametrize( + ("upstream_model", "reasoning_effort"), + ( + pytest.param("claude-haiku-4-5", "turbo", id="unknown-value"), + pytest.param("claude-opus-4-6", "xhigh", id="level-the-model-lacks"), + ), +) +def test_unsupported_reasoning_effort_is_rejected_before_reaching_anthropic( + gateway: Gateway, upstream_model: str, reasoning_effort: str +) -> None: + with wire_server(lambda request: Reply()) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{upstream_model}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY + ) + response: Final = gateway.request( + "POST", "/v1/messages", {**_claude_code_turn({"reasoning_effort": reasoning_effort}), "model": model} + ) + assert response.status_code == 400, response.text + assert response.json()["error"]["type"] == "invalid_request_error", response.text + assert wire.drain() == () diff --git a/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_response_wire.py b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_response_wire.py new file mode 100644 index 00000000000..1ada28b3355 --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_response_wire.py @@ -0,0 +1,63 @@ +import uuid +from typing import Final + +from integration._support import claude_code as cc +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +_MODEL: Final = "claude-haiku-4-5" +_ANTHROPIC_CONTENT: Final = ( + {"type": "thinking", "thinking": "the user wants a single word", "signature": "EqQBCkgIBRABGAIiQLz"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzixlit"}, + {"type": "text", "text": "PONG"}, +) +_ANTHROPIC_USAGE: Final = {"input_tokens": 12, "output_tokens": 30, "output_tokens_details": {"thinking_tokens": 20}} + + +def _client_body(stream: bool) -> dict[str, JsonValue]: + return { + **cc.claude_code_request(f"cache-bust-{uuid.uuid4().hex}"), + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "stream": stream, + } + + +def test_streamed_thinking_blocks_and_thinking_token_count_reach_the_client_unchanged(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + return Reply( + chunks=cc.message_stream(f"msg_{uuid.uuid4().hex}", _MODEL, _ANTHROPIC_CONTENT, _ANTHROPIC_USAGE), + content_type="text/event-stream", + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY) + response: Final = gateway.request("POST", "/v1/messages", {**_client_body(stream=True), "model": model}) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + assert cc.streamed_content(response.text) == [ + {"type": "thinking", "thinking": "the user wants a single word", "signature": "EqQBCkgIBRABGAIiQLz"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzixlit"}, + {"type": "text", "text": "PONG"}, + ], response.text + assert cc.streamed_usage(response.text) == { + "output_tokens": 30, + "output_tokens_details": {"thinking_tokens": 20}, + }, response.text + + +def test_non_streamed_thinking_blocks_and_thinking_token_count_reach_the_client_unchanged(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + return Reply(body=cc.message_reply(f"msg_{uuid.uuid4().hex}", _MODEL, _ANTHROPIC_CONTENT, _ANTHROPIC_USAGE)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY) + response: Final = gateway.request("POST", "/v1/messages", {**_client_body(stream=False), "model": model}) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + assert response.json()["content"] == [ + {"type": "thinking", "thinking": "the user wants a single word", "signature": "EqQBCkgIBRABGAIiQLz"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzixlit"}, + {"type": "text", "text": "PONG"}, + ], response.text + assert response.json()["usage"]["output_tokens_details"] == {"thinking_tokens": 20}, response.text diff --git a/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_token_pricing_wire.py b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_token_pricing_wire.py new file mode 100644 index 00000000000..cf3c709028c --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/reasoning/test_anthropic_reasoning_token_pricing_wire.py @@ -0,0 +1,64 @@ +import uuid +from typing import Final + +import pytest +from integration._support import claude_code as cc +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server + +_MODEL: Final = "claude-haiku-4-5" +_INPUT_RATE: Final = 1e-6 +_OUTPUT_RATE: Final = 2e-6 +_REASONING_RATE: Final = 7e-6 +_CONTENT: Final = ( + {"type": "thinking", "thinking": "count the words", "signature": "EqQBCkgIBRABGAIiQLz"}, + {"type": "text", "text": "PONG"}, +) +_USAGE: Final = {"input_tokens": 100, "output_tokens": 50, "output_tokens_details": {"thinking_tokens": 30}} + + +def _reply(identity: str, stream: bool) -> Reply: + if stream: + return Reply(chunks=cc.message_stream(identity, _MODEL, _CONTENT, _USAGE), content_type="text/event-stream") + return Reply(body=cc.message_reply(identity, _MODEL, _CONTENT, _USAGE)) + + +@pytest.mark.parametrize("stream", (pytest.param(False, id="non-streamed"), pytest.param(True, id="streamed"))) +def test_reported_thinking_tokens_are_billed_at_the_reasoning_rate_and_the_rest_at_the_output_rate( + gateway: Gateway, stream: bool +) -> None: + identity: Final = f"msg_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + return _reply(identity, stream) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{_MODEL}", + api_base=wire.url, + api_key=cc.ANTHROPIC_API_KEY, + input_cost_per_token=_INPUT_RATE, + output_cost_per_token=_OUTPUT_RATE, + output_cost_per_reasoning_token=_REASONING_RATE, + ) + body: Final = { + **cc.claude_code_request(f"cache-bust-{uuid.uuid4().hex}"), + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "stream": stream, + "model": model, + } + response: Final = gateway.request("POST", "/v1/messages", body) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (identity,)), + lambda values: len(values) == 1, + seconds=70, + ) + input_tokens, output_tokens, thinking_tokens = 100, 50, 30 + assert float(rows[0]["spend"]) == pytest.approx( + input_tokens * _INPUT_RATE + + (output_tokens - thinking_tokens) * _OUTPUT_RATE + + thinking_tokens * _REASONING_RATE + ), rows From c2ae4837824e7320ab94ed3bf67ebad29c4e76ef Mon Sep 17 00:00:00 2001 From: shrey-berri Date: Thu, 1 Oct 2026 09:21:10 -0700 Subject: [PATCH 005/171] fix(bedrock): add beta for thinking display updates (#43832) --- litellm/llms/anthropic/common_utils.py | 29 +- .../pass_through/messages/transformation.py | 12 +- .../anthropic_claude3_transformation.py | 15 + .../anthropic_claude3_transformation.py | 26 +- litellm/types/llms/anthropic.py | 4 +- ...est_anthropic_messages_per_turn_control.py | 20 ++ .../anthropic/test_anthropic_common_utils.py | 76 ++++- ...ations_anthropic_claude3_transformation.py | 166 ++++++----- .../test_anthropic_claude3_transformation.py | 272 ++++++++---------- 9 files changed, 395 insertions(+), 225 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 2ba7e9b6657..09fe42e8fe5 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -34,9 +34,11 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_OAUTH_BETA_HEADER, ANTHROPIC_OAUTH_TOKEN_PREFIX, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, AllAnthropicToolsValues, AnthropicMcpServerTool, AnthropicMessagesToolChoice, + AnthropicThinkingParam, ) from litellm.types.llms.openai import AllMessageValues from litellm.types.proxy.model_listing import ModelInfoResponse @@ -337,6 +339,11 @@ class AnthropicModelInfo(BaseLLMModelInfo): file_ids: Final = get_file_ids_from_messages(messages) return len(file_ids) > 0 + def is_thinking_display_updates_used(self, thinking: AnthropicThinkingParam | None) -> bool: + if not isinstance(thinking, dict): + return False + return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates" + def is_mid_conversation_output_config_used(self, messages: list[AllMessageValues]) -> bool: """ Return if "output_config" is in a message @@ -749,7 +756,11 @@ class AnthropicModelInfo(BaseLLMModelInfo): custom_llm_provider=custom_llm_provider, ) existing_output_config: Final = optional_params.get("output_config") - optional_params["thinking"] = {"type": "adaptive"} + display: Final = thinking.get("display") + if display in ("summarized", "omitted"): + optional_params["thinking"] = {"type": "adaptive", "display": display} + else: + optional_params["thinking"] = {"type": "adaptive"} optional_params["output_config"] = { "effort": effort, **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), @@ -869,6 +880,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): *, custom_llm_provider: str, is_mid_conversation_output_config_used: bool = False, + is_thinking_display_updates_used: bool = False, ) -> list[str]: """ Get list of common beta headers based on the features that are active. @@ -904,7 +916,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_mid_conversation_output_config_used: betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER) - return list(set(betas)) + thinking_display_betas: Final = ( + (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else () + ) + return list(set(betas).union(thinking_display_betas)) @staticmethod def _make_api_key_auth_header(api_key: str, api_base: str | None, use_bearer_for_custom_base: bool = False) -> dict: @@ -937,6 +952,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): api_base: str | None = None, use_bearer_for_custom_base: bool = False, is_mid_conversation_output_config_used: bool = False, + is_thinking_display_updates_used: bool = False, ) -> dict: betas: Final = set() # Anthropic no longer requires the prompt-caching beta header @@ -993,6 +1009,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): if user_anthropic_beta_headers is not None: betas.update(user_anthropic_beta_headers) + all_betas: Final = betas.union( + (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else () + ) + # Don't send any beta headers to Vertex, except web search which is required if is_vertex_request is True: # Vertex AI requires web search beta header for web search to work @@ -1000,8 +1020,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value - elif len(betas) > 0: - headers["anthropic-beta"] = ",".join(betas) + elif len(all_betas) > 0: + headers["anthropic-beta"] = ",".join(all_betas) return headers @@ -1059,6 +1079,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): auth_token=auth_token, file_id_used=file_id_used, is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, + is_thinking_display_updates_used=self.is_thinking_display_updates_used(optional_params.get("thinking")), web_search_tool_used=web_search_tool_used, is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index 1f604cfb8d7..b1be92e49b6 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -12,6 +12,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, AnthropicMessagesRequest, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -688,8 +689,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if AnthropicModelInfo().is_tool_search_used(tools): beta_values.add(get_tool_search_beta_header(custom_llm_provider)) - if not beta_values: + thinking_display_betas: Final = ( + (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) + if AnthropicModelInfo().is_thinking_display_updates_used(optional_params.get("thinking")) + else () + ) + all_beta_values: Final = beta_values.union(thinking_display_betas) + + if not all_beta_values: return headers merged: Final = {key: value for key, value in headers.items() if key.lower() != "anthropic-beta"} - merged["anthropic-beta"] = ",".join(sorted(beta_values)) + merged["anthropic-beta"] = ",".join(sorted(all_beta_values)) return merged diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 4d758640d72..0be087d2dd4 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -29,6 +29,7 @@ from litellm.llms.bedrock.common_utils import ( from litellm.types.llms.anthropic import ( ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_BETA_HEADER, + AnthropicThinkingParam, ) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -85,6 +86,8 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): from litellm.utils import supports_native_structured_output original_model: Final = model + requested_thinking: Final = non_default_params.get("thinking") + requested_display_updates: Final = self.is_thinking_display_updates_used(requested_thinking) if "response_format" in non_default_params and not supports_native_structured_output( model=model, custom_llm_provider="bedrock" ): @@ -114,6 +117,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( model=original_model, optional_params=optional_params, custom_llm_provider="bedrock" ) + translated_thinking: Final = optional_params.get("thinking") + if ( + requested_display_updates + and isinstance(translated_thinking, dict) + and translated_thinking.get("type") == "adaptive" + ): + thinking_with_display: Final[AnthropicThinkingParam] = {"type": "adaptive", "display": "updates"} + optional_params["thinking"] = thinking_with_display # The stub model hides the original model from the parent's forced-tool-use backstop response_format_tool_choice: Final = optional_params.get("tool_choice") @@ -170,6 +181,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): messages=messages, optional_params=optional_params, headers=headers, + thinking=_anthropic_request.get("thinking"), ) if beta_list: _anthropic_request["anthropic_beta"] = beta_list @@ -250,12 +262,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, headers: dict, + thinking: AnthropicThinkingParam | None, ) -> list[str]: tools: Final = optional_params.get("tools") tool_search_used: Final = self.is_tool_search_used(tools) programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools) input_examples_used: Final = self.is_input_examples_used(tools) is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages) + is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking) user_beta_set: Final = set(get_anthropic_beta_from_headers(headers)) beta_set: Final = set(user_beta_set) @@ -268,6 +282,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, + is_thinking_display_updates_used=is_thinking_display_updates_used, ) beta_set.update(auto_betas) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index f018077ebcd..73da7c41a09 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -49,6 +49,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_BETA_HEADER, + AnthropicThinkingParam, ) from litellm.types.llms.bedrock import BedrockInvokeAnthropicMessagesRequest from litellm.types.llms.openai import AllMessageValues @@ -348,8 +349,18 @@ class AmazonAnthropicClaudeMessagesConfig( if not isinstance(output_config, dict): output_config = {} output_config.setdefault("effort", self._effort_from_thinking_budget(budget_tokens)) + thinking: Final = anthropic_messages_request.get("thinking") + display: Final = thinking.get("display") if isinstance(thinking, dict) else None anthropic_messages_request["output_config"] = output_config - anthropic_messages_request["thinking"] = {"type": "adaptive"} + if display is None: + adaptive_thinking: Final[AnthropicThinkingParam] = {"type": "adaptive"} + anthropic_messages_request["thinking"] = adaptive_thinking + else: + adaptive_thinking_with_display: Final[AnthropicThinkingParam] = { + "type": "adaptive", + "display": display, + } + anthropic_messages_request["thinking"] = adaptive_thinking_with_display verbose_logger.debug( "Bedrock clear_thinking_20251015: injected adaptive thinking with effort=%s for model=%s", output_config.get("effort"), @@ -535,6 +546,9 @@ class AmazonAnthropicClaudeMessagesConfig( ), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, + is_thinking_display_updates_used=anthropic_model_info.is_thinking_display_updates_used( + anthropic_messages_request.get("thinking") + ), ) beta_set.update(auto_betas) @@ -664,6 +678,8 @@ class AmazonAnthropicClaudeMessagesConfig( litellm_params: GenericLiteLLMParams, headers: dict, ) -> dict: + requested_thinking: Final = anthropic_messages_optional_request_params.get("thinking") + requested_display_updates: Final = AnthropicModelInfo().is_thinking_display_updates_used(requested_thinking) self._clamp_adaptive_reasoning_effort_for_bedrock( model=model, optional_params=anthropic_messages_optional_request_params, @@ -676,6 +692,14 @@ class AmazonAnthropicClaudeMessagesConfig( litellm_params=litellm_params, headers=headers, ) + translated_thinking: Final = anthropic_messages_request.get("thinking") + if ( + requested_display_updates + and isinstance(translated_thinking, dict) + and translated_thinking.get("type") == "adaptive" + ): + thinking_with_display: Final[AnthropicThinkingParam] = {"type": "adaptive", "display": "updates"} + anthropic_messages_request["thinking"] = thinking_with_display self._normalize_system_role_messages(anthropic_messages_request, model=model) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 60d5450a1f2..6cd0e55c517 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -733,7 +733,7 @@ ANTHROPIC_API_ONLY_HEADERS: Final = { # fails if calling anthropic on vertex ai class AnthropicThinkingParam(TypedDict, total=False): type: ReadOnly[Literal["enabled", "adaptive", "disabled"]] budget_tokens: int - display: ReadOnly[Literal["summarized", "omitted"]] + display: ReadOnly[Literal["summarized", "omitted", "updates"]] class ANTHROPIC_HOSTED_TOOLS(str, Enum): @@ -776,6 +776,8 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01" +ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18" + ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14" # OAuth constants diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py index ef6a6e72d07..ef1fac9e120 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py @@ -130,3 +130,23 @@ def test_json_provider_passthrough_adds_per_turn_control_beta(): ) assert PER_TURN_CONTROL in _betas(headers) + + +@pytest.mark.parametrize("display", (None, "summarized", "omitted", "updates")) +@pytest.mark.parametrize("explicit_beta", (False, True)) +def test_native_messages_thinking_display_updates_beta(display: str | None, explicit_beta: bool) -> None: + from typing import Final + + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + beta: Final = ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + headers, _ = AnthropicMessagesConfig().validate_anthropic_messages_environment( + headers={"anthropic-beta": beta} if explicit_beta else {}, + model="claude-opus-5", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params={"thinking": {"type": "adaptive", "display": display}} if display else {}, + litellm_params={}, + api_key="sk-ant-test", + ) + + assert headers.get("anthropic-beta", "").split(",").count(beta) == int(display == "updates" or explicit_beta) diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index b80129a55bf..0904a20a16b 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -1984,7 +1984,6 @@ class TestClaudeOpus48AdaptiveThinking: assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True - @pytest.mark.parametrize( "model", [ @@ -2376,3 +2375,78 @@ def test_validate_environment_adds_mid_conversation_output_config_beta( assert headers.get("anthropic-beta", "").split(",").count(beta) == int(nested_output_config or explicit_beta) assert headers["x-api-key"] == FAKE_REGULAR_KEY + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("display", (None, "summarized", "omitted", "updates")) +@pytest.mark.parametrize("explicit_beta", (False, True)) +def test_validate_environment_adds_thinking_display_updates_beta(display: str | None, explicit_beta: bool) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + beta: Final = ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + headers: Final = AnthropicModelInfo().validate_environment( + headers={"anthropic-beta": beta} if explicit_beta else {}, + model="claude-opus-5", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params={"thinking": {"type": "adaptive", "display": display}} if display else {}, + litellm_params={}, + api_key=FAKE_REGULAR_KEY, + ) + + assert headers.get("anthropic-beta", "").split(",").count(beta) == int(display == "updates" or explicit_beta) + assert headers["x-api-key"] == FAKE_REGULAR_KEY + + +@pytest.mark.parametrize( + ("thinking", "expected"), + ( + (None, False), + ({}, False), + ("updates", False), + ({"display": "updates"}, False), + ({"type": "disabled", "display": "updates"}, False), + ({"type": "enabled", "display": "updates", "budget_tokens": 1024}, True), + ), +) +def test_thinking_display_beta_requires_active_thinking(thinking: object, expected: bool) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + headers: Final = AnthropicModelInfo().validate_environment( + headers={}, + model="claude-opus-5", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params={"thinking": thinking}, + litellm_params={}, + api_key=FAKE_REGULAR_KEY, + ) + + assert (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in headers.get("anthropic-beta", "").split(",")) is expected + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("display", "expected_thinking"), + ( + ("summarized", {"type": "adaptive", "display": "summarized"}), + ("omitted", {"type": "adaptive", "display": "omitted"}), + ("updates", {"type": "adaptive"}), + ), +) +def test_shared_legacy_thinking_translation_preserves_supported_display( + display: str, expected_thinking: dict[str, str] +) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + optional_params: Final = { + "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}, + } + + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model="claude-opus-5", + optional_params=optional_params, + custom_llm_provider="azure_ai", + ) + + assert optional_params["thinking"] == expected_thinking diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 5d97beeb3fc..2c82ba1c5b8 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -1,4 +1,3 @@ -import asyncio import base64 import copy import json @@ -12,14 +11,12 @@ import pytest # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. - import litellm from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - ONE_PIXEL_PNG = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" ) @@ -93,9 +90,7 @@ def local_beta_headers_config(monkeypatch): def test_get_supported_params_thinking(): config = AmazonAnthropicClaudeConfig() - params = config.get_supported_openai_params( - model="anthropic.claude-sonnet-4-20250514-v1:0" - ) + params = config.get_supported_openai_params(model="anthropic.claude-sonnet-4-20250514-v1:0") assert "thinking" in params @@ -148,53 +143,23 @@ def test_aws_params_filtered_from_request_body(): result_json = json.dumps(result) # Verify AWS authentication params are NOT in the request body - assert ( - "aws_access_key_id" not in result_json - ), "AWS access key should not be in request body" - assert ( - "aws_secret_access_key" not in result_json - ), "AWS secret key should not be in request body" - assert ( - "aws_session_token" not in result_json - ), "AWS session token should not be in request body" - assert ( - "aws_region_name" not in result_json - ), "AWS region should not be in request body" - assert ( - "aws_role_name" not in result_json - ), "AWS role name should not be in request body" - assert ( - "aws_session_name" not in result_json - ), "AWS session name should not be in request body" - assert ( - "aws_profile_name" not in result_json - ), "AWS profile name should not be in request body" - assert ( - "aws_web_identity_token" not in result_json - ), "AWS web identity token should not be in request body" - assert ( - "aws_sts_endpoint" not in result_json - ), "AWS STS endpoint should not be in request body" - assert ( - "aws_bedrock_runtime_endpoint" not in result_json - ), "AWS bedrock endpoint should not be in request body" - assert ( - "aws_external_id" not in result_json - ), "AWS external ID should not be in request body" - assert ( - "aws_session_tags" not in result_json - ), "AWS session tags should not be in request body" + assert "aws_access_key_id" not in result_json, "AWS access key should not be in request body" + assert "aws_secret_access_key" not in result_json, "AWS secret key should not be in request body" + assert "aws_session_token" not in result_json, "AWS session token should not be in request body" + assert "aws_region_name" not in result_json, "AWS region should not be in request body" + assert "aws_role_name" not in result_json, "AWS role name should not be in request body" + assert "aws_session_name" not in result_json, "AWS session name should not be in request body" + assert "aws_profile_name" not in result_json, "AWS profile name should not be in request body" + assert "aws_web_identity_token" not in result_json, "AWS web identity token should not be in request body" + assert "aws_sts_endpoint" not in result_json, "AWS STS endpoint should not be in request body" + assert "aws_bedrock_runtime_endpoint" not in result_json, "AWS bedrock endpoint should not be in request body" + assert "aws_external_id" not in result_json, "AWS external ID should not be in request body" + assert "aws_session_tags" not in result_json, "AWS session tags should not be in request body" # Also check that the sensitive values themselves are not in the response - assert ( - "AKIAIOSFODNN7EXAMPLE" not in result_json - ), "AWS access key value leaked in request body" - assert ( - "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" not in result_json - ), "AWS secret key value leaked in request body" - assert ( - "arn:aws:iam::123456789012:role/test-role" not in result_json - ), "AWS role ARN leaked in request body" + assert "AKIAIOSFODNN7EXAMPLE" not in result_json, "AWS access key value leaked in request body" + assert "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" not in result_json, "AWS secret key value leaked in request body" + assert "arn:aws:iam::123456789012:role/test-role" not in result_json, "AWS role ARN leaked in request body" assert "test-session" not in result_json, "AWS session name leaked in request body" # Verify normal params ARE still in the request body @@ -203,9 +168,7 @@ def test_aws_params_filtered_from_request_body(): assert result["top_p"] == 0.9, "top_p should be in request body" # Verify Bedrock-specific params are added - assert ( - result["anthropic_version"] == "bedrock-2023-05-31" - ), "anthropic_version should be set" + assert result["anthropic_version"] == "bedrock-2023-05-31", "anthropic_version should be set" assert "model" not in result, "model should be removed for Bedrock Invoke API" assert "stream" not in result, "stream should be removed for Bedrock Invoke API" @@ -262,9 +225,7 @@ def test_output_format_conversion_to_inline_schema(): ) # Verify output_format was removed from the request - assert ( - "output_format" not in result - ), "output_format should be removed from request body" + assert "output_format" not in result, "output_format should be removed from request body" # Verify the schema was added to the last user message content assert "messages" in result @@ -415,9 +376,7 @@ def test_opus_4_5_model_detection(): ] for model in non_opus_4_5_models: - assert not config._is_claude_opus_4_5( - model - ), f"Should not detect {model} as Opus 4.5" + assert not config._is_claude_opus_4_5(model), f"Should not detect {model} as Opus 4.5" # def test_structured_outputs_beta_header_filtered_for_bedrock_invoke(): @@ -595,9 +554,7 @@ def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(local_mo ("anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_output_config_effort_normalized_for_bedrock_chat_invoke_request( - model, expected_effort -): +def test_output_config_effort_normalized_for_bedrock_chat_invoke_request(model, expected_effort): """Bedrock Invoke chat path accepts ``xhigh`` and forwards the provider-safe effort.""" config = AmazonAnthropicClaudeConfig() @@ -668,9 +625,9 @@ def test_output_format_removed_from_bedrock_invoke_request(): ) # Verify output_format is not in the request - assert ( - "output_format" not in result - ), f"output_format should be removed for Bedrock Invoke, got keys: {result.keys()}" + assert "output_format" not in result, ( + f"output_format should be removed for Bedrock Invoke, got keys: {result.keys()}" + ) def test_bedrock_chat_invoke_forwards_output_config_format_natively(local_model_cost_map): @@ -866,7 +823,9 @@ async def test_bedrock_invoke_claude_async_completion_inlines_remote_images_off_ assert async_only_image_fetch.base64_png in captured["body"] -async def test_bedrock_invoke_claude_async_completion_inlines_document_url_sources_off_the_event_loop(async_only_image_fetch): +async def test_bedrock_invoke_claude_async_completion_inlines_document_url_sources_off_the_event_loop( + async_only_image_fetch, +): pdf_url = f"http://docs.example/{uuid.uuid4()}.pdf" captured = {} @@ -958,6 +917,62 @@ def test_bedrock_chat_invoke_tool_search_beta_follows_model_map( assert result.get("anthropic_beta") == expected_betas +def test_bedrock_chat_invoke_adds_thinking_display_updates_beta( + local_model_cost_map, local_beta_headers_config +) -> None: + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + config: Final = AmazonAnthropicClaudeConfig() + model: Final = "us.anthropic.claude-opus-5" + optional_params: Final = config.map_openai_params( + non_default_params={ + "max_tokens": 512, + "thinking": {"type": "adaptive", "display": "updates"}, + }, + optional_params={}, + model=model, + drop_params=False, + ) + result: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive", "display": "updates"} + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in result.get("anthropic_beta", []) + + +def test_bedrock_chat_invoke_preserves_display_when_translating_legacy_thinking( + local_model_cost_map, local_beta_headers_config +) -> None: + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + config: Final = AmazonAnthropicClaudeConfig() + model: Final = "us.anthropic.claude-opus-5" + optional_params: Final = config.map_openai_params( + non_default_params={ + "max_tokens": 512, + "thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}, + }, + optional_params={}, + model=model, + drop_params=False, + ) + result: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive", "display": "updates"} + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in result.get("anthropic_beta", []) + + FINE_GRAINED_TOOL_STREAMING_BETA: Final = "fine-grained-tool-streaming-2025-05-14" EAGER_TOOL_SCHEMA: Final = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]} @@ -1015,7 +1030,10 @@ def test_bedrock_chat_invoke_eager_input_streaming_beta_not_duplicated_with_clie def _mid_conversation_system_conversation() -> list[dict]: return [ - {"role": "system", "content": [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "system", + "content": [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": "First question"}, {"role": "assistant", "content": "First answer"}, {"role": "user", "content": "Second question"}, @@ -1074,7 +1092,11 @@ def _preserved_thinking_turns(reminder_after_user: bool) -> tuple[list[dict], li second_question = {"role": "user", "content": "Second question"} second_turn = [second_question, reminder] if reminder_after_user else [reminder, second_question] turn_n_plus_one = [*turn_n, _thinking_reply("First answer"), *second_turn] - turn_n_plus_two = [*turn_n_plus_one, _thinking_reply("Second answer"), {"role": "user", "content": "Third question"}] + turn_n_plus_two = [ + *turn_n_plus_one, + _thinking_reply("Second answer"), + {"role": "user", "content": "Third question"}, + ] return turn_n, turn_n_plus_one, turn_n_plus_two @@ -1102,7 +1124,11 @@ def test_chat_flagged_model_replays_a_byte_identical_prefix_around_a_mid_convers request must be a byte-identical prefix of turn N+1's or the block is dropped.""" requests = [ AmazonAnthropicClaudeConfig().transform_request( - model="invoke/us.anthropic.claude-fable-5-1", messages=copy.deepcopy(turn), optional_params={}, litellm_params={}, headers={} + model="invoke/us.anthropic.claude-fable-5-1", + messages=copy.deepcopy(turn), + optional_params={}, + litellm_params={}, + headers={}, ) for turn in _preserved_thinking_turns(reminder_after_user) ] diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index f92de7370bd..d7f451dd6ee 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -5,33 +5,33 @@ import json import os import struct import zlib +from collections.abc import AsyncIterator, Mapping, Sequence from datetime import datetime from types import SimpleNamespace -from collections.abc import AsyncIterator, Mapping, Sequence from typing import Final from unittest.mock import Mock import httpx import pytest -# Ensure the project root is on the import path so `litellm` can be imported when -# tests are executed from any working directory. - -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.bedrock.common_utils import ( - ensure_bedrock_anthropic_messages_tool_names, - normalize_custom_field_on_tools, - normalize_tool_input_schema_types_for_bedrock_invoke, -) from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) + +# Ensure the project root is on the import path so `litellm` can be imported when +# tests are executed from any working directory. +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( as_system_content_blocks, ) +from litellm.llms.bedrock.common_utils import ( + ensure_bedrock_anthropic_messages_tool_names, + normalize_custom_field_on_tools, + normalize_tool_input_schema_types_for_bedrock_invoke, +) from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, AmazonAnthropicClaudeMessagesStreamDecoder, @@ -54,9 +54,7 @@ async def test_bedrock_sse_wrapper_encodes_dict_chunks(): _dummy_stream(), litellm_logging_obj=LiteLLMLoggingObj( model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", - messages=[ - {"role": "user", "content": "Hello, can you tell me a short joke?"} - ], + messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}], stream=True, call_type="chat", start_time=datetime.now(), @@ -233,9 +231,7 @@ async def test_bedrock_sse_wrapper_keeps_usage_in_message_start_and_message_delt def test_chunk_parser_usage_transformation(): """Ensure Bedrock invocation metrics are transformed to Anthropic usage keys.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0") chunk = { "type": "message_delta", @@ -264,9 +260,7 @@ def test_chunk_parser_preserves_cache_usage_fields_with_invocation_metrics(): fields and cache tokens end up billed at $0. """ - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-sonnet-4-6" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") chunk = { "type": "message_stop", @@ -292,9 +286,7 @@ def test_chunk_parser_preserves_cache_usage_fields_with_invocation_metrics(): def test_chunk_parser_maps_cache_token_counts_from_invocation_metrics(): """Cache itemization inside invocationMetrics maps to Anthropic usage keys.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-sonnet-4-6" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") chunk = { "type": "message_stop", @@ -317,9 +309,7 @@ def test_chunk_parser_maps_cache_token_counts_from_invocation_metrics(): def test_chunk_parser_keeps_existing_token_counts_over_invocation_metrics(): """Token counts reported in the chunk's own usage block win over invocationMetrics.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-sonnet-4-6" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") chunk = { "type": "message_stop", @@ -354,9 +344,7 @@ async def test_bedrock_sse_wrapper_preserves_cache_usage_with_invocation_metrics final usage billed cache reads and writes at $0. """ - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-sonnet-4-6" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") cfg = AmazonAnthropicClaudeMessagesConfig() raw_chunks = [ @@ -566,11 +554,7 @@ def test_normalize_custom_field_on_tools(): assert request4["tools"] is None # Case 5: an explicit top-level flag wins over a conflicting wrapped one - request5 = { - "tools": [ - {"name": "Read", "defer_loading": False, "custom": {"defer_loading": True}} - ] - } + request5 = {"tools": [{"name": "Read", "defer_loading": False, "custom": {"defer_loading": True}}]} normalize_custom_field_on_tools(request5) assert request5["tools"][0] == {"name": "Read", "defer_loading": False} @@ -591,9 +575,7 @@ def test_normalize_custom_field_on_tools(): assert request7["tools"] == [{"name": "Read"}, {"name": "Write"}] -@pytest.mark.parametrize( - "deferred_marker", [{"custom": {"defer_loading": True}}, {"defer_loading": True}] -) +@pytest.mark.parametrize("deferred_marker", [{"custom": {"defer_loading": True}}, {"defer_loading": True}]) def test_bedrock_invoke_messages_transform_emits_top_level_defer_loading( deferred_marker, ): @@ -726,9 +708,7 @@ def test_bedrock_invoke_messages_skips_thinking_injection_when_already_enabled( "max_tokens": 32000, "stream": False, "thinking": {"type": "enabled", "budget_tokens": 2048}, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } result = cfg.transform_anthropic_messages_request( model="global.anthropic.claude-sonnet-4-6-v1:0", @@ -830,9 +810,7 @@ def test_remove_ttl_from_cache_control_processes_tools(local_model_cost_map): "messages": [], } - cfg._remove_ttl_from_cache_control( - request, model="anthropic.claude-3-5-sonnet-20241022-v2:0" - ) + cfg._remove_ttl_from_cache_control(request, model="anthropic.claude-3-5-sonnet-20241022-v2:0") # Tool ttl should be stripped assert "ttl" not in request["tools"][0]["cache_control"] @@ -868,9 +846,7 @@ def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5(local_ ], } - cfg._remove_ttl_from_cache_control( - request, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + cfg._remove_ttl_from_cache_control(request, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0") # Both tools and system should preserve ttl for Claude 4.5 assert request["tools"][0]["cache_control"]["ttl"] == "1h" @@ -954,9 +930,7 @@ def test_bedrock_messages_strips_output_config(): headers={}, ) - assert "output_config" not in result, ( - "output_config should be stripped for models that don't support it" - ) + assert "output_config" not in result, "output_config should be stripped for models that don't support it" assert result.get("max_tokens") == 4096 @@ -989,9 +963,7 @@ def test_bedrock_messages_preserves_output_config_for_claude_4_6(): headers={}, ) - assert "output_config" in result, ( - "output_config should be preserved for supported models" - ) + assert "output_config" in result, "output_config should be preserved for supported models" assert result["output_config"] == {"effort": "high"} assert result.get("max_tokens") == 4096 @@ -1143,9 +1115,7 @@ def test_bedrock_messages_converts_output_config_format_to_inline_schema(): ("anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bedrock_messages_normalizes_output_config_effort_for_opus( - model, expected_effort -): +def test_bedrock_messages_normalizes_output_config_effort_for_opus(model, expected_effort): """Bedrock /v1/messages accepts ``xhigh`` and forwards the provider-safe effort.""" from unittest.mock import patch @@ -1203,9 +1173,7 @@ def test_bedrock_messages_does_not_mutate_callers_messages_when_embedding_schema headers={}, ) - assert caller_messages == [ - {"role": "user", "content": [{"type": "text", "text": "Hello"}]} - ] + assert caller_messages == [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] assert caller_message == { "role": "user", "content": [{"type": "text", "text": "Hello"}], @@ -1521,9 +1489,7 @@ def test_bedrock_messages_strips_context_management(): messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] optional_params = { "max_tokens": 4096, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } result = cfg.transform_anthropic_messages_request( @@ -1534,9 +1500,7 @@ def test_bedrock_messages_strips_context_management(): headers={}, ) - assert "context_management" not in result, ( - "context_management should be stripped — Bedrock Invoke rejects it" - ) + assert "context_management" not in result, "context_management should be stripped — Bedrock Invoke rejects it" assert result.get("max_tokens") == 4096 @@ -1661,7 +1625,9 @@ def test_bedrock_messages_allowlist_filters_anthropic_only_fields(): ["dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14", "interleaved-thinking-2025-05-14"], ids=["client_sends_beta", "client_omits_beta"], ) -def test_bedrock_messages_forwards_safeguards_with_dangerous_tool_use_beta(local_beta_headers_config, client_beta_header): +def test_bedrock_messages_forwards_safeguards_with_dangerous_tool_use_beta( + local_beta_headers_config, client_beta_header +): """ Claude Code's server-side auto-mode classifier sends `safeguards` alongside the dangerous-tool-use-2026-09-03 beta. Bedrock Invoke accepts the pair, answers @@ -1769,12 +1735,8 @@ def test_bedrock_messages_filters_user_provided_unsupported_beta_header(): ) betas = result.get("anthropic_beta") or [] - assert "advisor-tool-2026-03-01" not in betas, ( - "user-provided beta not in the Bedrock mapping must be dropped" - ) - assert "context-1m-2025-08-07" in betas, ( - "user-provided beta that IS in the Bedrock mapping should survive" - ) + assert "advisor-tool-2026-03-01" not in betas, "user-provided beta not in the Bedrock mapping must be dropped" + assert "context-1m-2025-08-07" in betas, "user-provided beta that IS in the Bedrock mapping should survive" def test_bedrock_messages_renames_user_provided_aliased_beta_header(): @@ -1802,9 +1764,7 @@ def test_bedrock_messages_renames_user_provided_aliased_beta_header(): assert "advanced-tool-use-2025-11-20" not in betas, ( "Anthropic-direct spelling should be rewritten, not forwarded verbatim" ) - assert "tool-search-tool-2025-10-19" in betas, ( - "user-provided beta should be renamed to the Bedrock-side spelling" - ) + assert "tool-search-tool-2025-10-19" in betas, "user-provided beta should be renamed to the Bedrock-side spelling" @pytest.mark.asyncio @@ -2066,9 +2026,7 @@ async def test_unified_bedrock_messages_sse_usage_and_cost_claude_sonnet_46(): "global.anthropic.claude-fable-5", ], ) -def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models( - local_model_cost_map, model -): +def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models(local_model_cost_map, model): """clear_thinking_20251015 without a top-level ``thinking`` field must inject ``thinking.type=adaptive`` plus ``output_config.effort`` on adaptive-thinking models (Opus 4.7/4.8, Fable 5). The legacy ``thinking.type=enabled`` shape is @@ -2078,9 +2036,7 @@ def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models cfg = AmazonAnthropicClaudeMessagesConfig() request = { "max_tokens": 32000, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2103,9 +2059,7 @@ def test_bedrock_clear_thinking_converts_legacy_enabled_budget_to_effort(): "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, }, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2123,10 +2077,7 @@ def test_resolve_clear_thinking_budget_tokens_honors_explicit_zero(): and only fall back to the minimum when the caller omits the budget.""" cfg = AmazonAnthropicClaudeMessagesConfig() assert cfg._resolve_clear_thinking_budget_tokens(0) == 0 - assert ( - cfg._resolve_clear_thinking_budget_tokens(None) - == BEDROCK_MIN_THINKING_BUDGET_TOKENS - ) + assert cfg._resolve_clear_thinking_budget_tokens(None) == BEDROCK_MIN_THINKING_BUDGET_TOKENS assert cfg._resolve_clear_thinking_budget_tokens(12000) == 12000 @@ -2136,9 +2087,7 @@ def test_bedrock_clear_thinking_keeps_enabled_for_non_adaptive_models(): cfg = AmazonAnthropicClaudeMessagesConfig() request = { "max_tokens": 32000, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2163,9 +2112,7 @@ def test_bedrock_invoke_transform_emits_adaptive_thinking_for_opus_4_8(): optional_params = { "max_tokens": 32000, "stream": False, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } result = cfg.transform_anthropic_messages_request( @@ -2202,9 +2149,7 @@ def test_bedrock_invoke_transform_normalizes_system_role_message_into_system(): assert all(m.get("role") != "system" for m in result["messages"]) assert result["messages"] == [{"role": "user", "content": "hi"}] - assert result["system"] == [ - {"type": "text", "text": "You are a careful assistant."} - ] + assert result["system"] == [{"type": "text", "text": "You are a careful assistant."}] def test_bedrock_invoke_transform_merges_system_role_into_existing_system(): @@ -2319,9 +2264,7 @@ def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(lo ) assert result["messages"] == messages - assert result["system"] == [ - {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}} - ] + assert result["system"] == [{"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}}] def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cost_map): @@ -2504,13 +2447,13 @@ def test_bedrock_invoke_transform_converted_system_carries_only_its_content(loca assert result["messages"][2] == { "role": "user", "content": [ - { - "type": "text", - "text": ( - "Operator note (not from the user): the following was " - "originally a mid-conversation system-role reminder." - ), - }, + { + "type": "text", + "text": ( + "Operator note (not from the user): the following was " + "originally a mid-conversation system-role reminder." + ), + }, {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, ], } @@ -2646,10 +2589,7 @@ def test_as_system_content_blocks_handles_each_shape(): def test_effort_from_thinking_budget_tiers(budget_tokens, expected_effort): """The budget -> effort mapping pins each tier boundary so a shifted threshold is caught.""" - assert ( - AmazonAnthropicClaudeMessagesConfig._effort_from_thinking_budget(budget_tokens) - == expected_effort - ) + assert AmazonAnthropicClaudeMessagesConfig._effort_from_thinking_budget(budget_tokens) == expected_effort def test_inject_adaptive_thinking_preserves_existing_effort(): @@ -2658,9 +2598,7 @@ def test_inject_adaptive_thinking_preserves_existing_effort(): cfg = AmazonAnthropicClaudeMessagesConfig() request = {"output_config": {"effort": "max", "other": "keep"}} - cfg._inject_adaptive_thinking_for_clear_thinking( - request, budget_tokens=24000, model="us.anthropic.claude-fable-5" - ) + cfg._inject_adaptive_thinking_for_clear_thinking(request, budget_tokens=24000, model="us.anthropic.claude-fable-5") assert request["thinking"] == {"type": "adaptive"} assert request["output_config"] == {"effort": "max", "other": "keep"} @@ -2673,9 +2611,7 @@ def test_bedrock_clear_thinking_noops_when_thinking_already_adaptive(): request = { "max_tokens": 32000, "thinking": {"type": "adaptive"}, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2695,9 +2631,7 @@ def test_bedrock_clear_thinking_replaces_disabled_thinking_on_adaptive_model(): request = { "max_tokens": 32000, "thinking": {"type": "disabled"}, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2717,9 +2651,7 @@ def test_bedrock_clear_thinking_leaves_enabled_thinking_on_non_adaptive_model(): request = { "max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 8000}, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2754,9 +2686,7 @@ def test_bedrock_messages_preserves_clear_tool_uses_context_management_and_adds_ messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] optional_params = { "max_tokens": 4096, - "context_management": { - "edits": [{"type": "clear_tool_uses_20250919"}] - }, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, } result = cfg.transform_anthropic_messages_request( @@ -2767,12 +2697,11 @@ def test_bedrock_messages_preserves_clear_tool_uses_context_management_and_adds_ headers={}, ) - assert result.get("context_management") == { - "edits": [{"type": "clear_tool_uses_20250919"}] - }, "clear_tool_uses_20250919 edit must reach Bedrock InvokeModel body" + assert result.get("context_management") == {"edits": [{"type": "clear_tool_uses_20250919"}]}, ( + "clear_tool_uses_20250919 edit must reach Bedrock InvokeModel body" + ) assert "context-management-2025-06-27" in result.get("anthropic_beta", []), ( - "context-management-2025-06-27 beta must reach the InvokeModel body so " - "the tool-call-clearing edit is accepted" + "context-management-2025-06-27 beta must reach the InvokeModel body so the tool-call-clearing edit is accepted" ) @@ -2849,9 +2778,9 @@ def test_bedrock_messages_filters_clear_thinking_keeps_clear_tool_uses( cm = result.get("context_management") assert cm is not None - assert [e.get("type") for e in cm["edits"]] == [ - "clear_tool_uses_20250919" - ], "clear_thinking_20251015 must still be stripped (LiteLLM-internal)" + assert [e.get("type") for e in cm["edits"]] == ["clear_tool_uses_20250919"], ( + "clear_thinking_20251015 must still be stripped (LiteLLM-internal)" + ) betas = result.get("anthropic_beta", []) assert "context-management-2025-06-27" in betas @@ -2992,9 +2921,7 @@ def test_bedrock_messages_tool_search_follows_claude_tool_search_rule(local_mode assert cfg._supports_tool_search_on_bedrock(model) is expected -def test_bedrock_messages_thinking_shape_follows_exact_bedrock_entry_flag( - local_model_cost_map, monkeypatch -): +def test_bedrock_messages_thinking_shape_follows_exact_bedrock_entry_flag(local_model_cost_map, monkeypatch): """The outbound thinking payload must follow the exact Bedrock cost-map entry. Before threading the caller's provider through the capability probes, the probe was pinned to ``"anthropic"``: the exact ``global.anthropic.claude-opus-4-8`` @@ -3002,7 +2929,6 @@ def test_bedrock_messages_thinking_shape_follows_exact_bedrock_entry_flag( forced ``thinking.type='adaptive'`` even with ``supports_adaptive_thinking`` explicitly set to ``false`` on the entry.""" import litellm - from litellm.types.router import GenericLiteLLMParams model = "global.anthropic.claude-opus-4-8" @@ -3404,22 +3330,14 @@ def test_bedrock_invoke_eager_input_streaming_beta_not_duplicated_with_client_he def _bedrock_event_frame(payload: Mapping[str, object]) -> bytes: def _header(name: str, value: str) -> bytes: - return ( - bytes([len(name)]) - + name.encode() - + bytes([7]) - + struct.pack(">H", len(value)) - + value.encode() - ) + return bytes([len(name)]) + name.encode() + bytes([7]) + struct.pack(">H", len(value)) + value.encode() headers: Final = ( _header(":message-type", "event") + _header(":event-type", "chunk") + _header(":content-type", "application/json") ) - body: Final = json.dumps( - {"bytes": base64.b64encode(json.dumps(payload).encode()).decode()} - ).encode() + body: Final = json.dumps({"bytes": base64.b64encode(json.dumps(payload).encode()).decode()}).encode() prelude: Final = struct.pack(">II", 12 + len(headers) + len(body) + 4, len(headers)) prelude_crc: Final = struct.pack(">I", zlib.crc32(prelude)) message_crc: Final = struct.pack(">I", zlib.crc32(prelude + prelude_crc + headers + body)) @@ -3547,3 +3465,65 @@ def test_bedrock_messages_removed_output_config_does_not_add_beta(explicit_beta: assert result["messages"] == [{"role": "user", "content": "Reply with OK"}] assert result.get("anthropic_beta", []).count(beta) == int(explicit_beta) + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("display", (None, "summarized", "omitted", "updates")) +@pytest.mark.parametrize("explicit_beta", (False, True)) +def test_bedrock_messages_thinking_display_updates_beta(display: str | None, explicit_beta: bool) -> None: + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + from litellm.types.router import GenericLiteLLMParams + + beta: Final = ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + thinking: Final = {"type": "adaptive", "display": display} if display else None + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="eu.anthropic.claude-opus-5", + messages=[{"role": "user", "content": "Reply with OK"}], + anthropic_messages_optional_request_params={"max_tokens": 512, **({"thinking": thinking} if thinking else {})}, + litellm_params=GenericLiteLLMParams(), + headers={"anthropic-beta": beta} if explicit_beta else {}, + ) + + assert result.get("anthropic_beta", []).count(beta) == int(display == "updates" or explicit_beta) + assert result.get("thinking") == thinking + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +def test_bedrock_messages_preserves_display_when_translating_legacy_thinking() -> None: + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + from litellm.types.router import GenericLiteLLMParams + + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="eu.anthropic.claude-opus-5", + messages=[{"role": "user", "content": "Reply with OK"}], + anthropic_messages_optional_request_params={ + "max_tokens": 512, + "thinking": {"type": "enabled", "budget_tokens": 24000, "display": "updates"}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive", "display": "updates"} + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in result.get("anthropic_beta", []) + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +def test_bedrock_clear_thinking_preserves_display_updates() -> None: + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + from litellm.types.router import GenericLiteLLMParams + + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="us.anthropic.claude-opus-4-6", + messages=[{"role": "user", "content": "Reply with OK"}], + anthropic_messages_optional_request_params={ + "max_tokens": 512, + "thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}, + "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive", "display": "updates"} + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in result.get("anthropic_beta", []) From d96477abce06f7591be19b2429db13ac3e955aa0 Mon Sep 17 00:00:00 2001 From: shrey-berri Date: Thu, 1 Oct 2026 09:23:33 -0700 Subject: [PATCH 006/171] fix(proxy): preserve decision request bodies under token limits (#43920) --- .../hooks/parallel_request_limiter_v3.py | 8 ++ .../pass_through_endpoints.py | 9 ++- litellm/proxy/utils.py | 24 ++++-- .../pass_through_endpoints.py | 1 + .../test_llm_pass_through_endpoints.py | 78 ++++++++++++++++++- .../test_pass_through_endpoints.py | 15 +++- 6 files changed, 123 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 36896bd6d44..c8fe49afcc9 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -75,6 +75,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import ( CallTypes, EmbeddingResponse, @@ -978,6 +979,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_limit: int | None, call_type: str | None, configured_output_tokens: int | None = None, + endpoint_type: EndpointType = EndpointType.GENERIC, ) -> None: """Hard-cap generation length when the request has no explicit cap. @@ -1006,6 +1008,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): capped_floor >= baseline_floor or _PROXY_MaxParallelRequestsHandler_v3._has_explicit_output_cap(data, call_type) or is_embedding + or endpoint_type == EndpointType.DECISIONS ): return effective_cap: Final = max(capped_floor, configured_output_tokens or 0) @@ -3757,6 +3760,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): tpm_reservation_scopes: Sequence[tuple[str, str]], tpm_reservation_amount: int, call_type: str | None = None, + endpoint_type: EndpointType = EndpointType.GENERIC, ) -> None: """ Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style @@ -3810,6 +3814,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): data=data, min_configured_limit=min_configured_otpm_limit, call_type=call_type, + endpoint_type=endpoint_type, ) io_response, itpm_reserved, otpm_reserved = await self.reserve_io_tokens( @@ -3984,6 +3989,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): cache: DualCache, data: dict, call_type: str, + endpoint_type: EndpointType = EndpointType.GENERIC, ): """ Pre-call hook to check rate limits before making the API call. @@ -4115,6 +4121,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_limit=min_configured_tpm_limit, call_type=call_type, configured_output_tokens=configured_output_tokens, + endpoint_type=endpoint_type, ) # Floor at 1 token so contentless requests (/responses, @@ -4201,6 +4208,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): tpm_reservation_scopes=tpm_reservation_scopes, tpm_reservation_amount=tpm_reservation_amount, call_type=call_type, + endpoint_type=endpoint_type, ) def _create_pipeline_operations( diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 2e7f9c4a41c..d15a14bb4b2 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -377,8 +377,10 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): return return_headers @staticmethod - def get_endpoint_type(url: str) -> EndpointType: + def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> EndpointType: parsed_url: Final = urlparse(url) + if custom_llm_provider == "typesafe" and parsed_url.path.removesuffix("/").endswith("/v1/systemone"): + return EndpointType.DECISIONS if ( ("generateContent") in url or ("streamGenerateContent") in url @@ -1093,7 +1095,9 @@ async def pass_through_request( requested_query_params: dict | None = query_params or dict(request.query_params) or None - endpoint_type: Final[EndpointType] = HttpPassThroughEndpointHelpers.get_endpoint_type(str(url)) + endpoint_type: Final[EndpointType] = HttpPassThroughEndpointHelpers.get_endpoint_type( + str(url), custom_llm_provider + ) # SigV4-signed callers (e.g. Bedrock) attach the exact bytes that were # signed via request.state; we must send those instead of re-encoding the @@ -1180,6 +1184,7 @@ async def pass_through_request( user_api_key_dict=user_api_key_dict, data=_parsed_body, call_type="pass_through_endpoint", + endpoint_type=endpoint_type, ) resolved_timeout: Final = resolve_pass_through_request_timeout(timeout) async_client_obj: Final = get_async_httpx_client( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c0e36e6e172..0af4a5cce8a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -245,6 +245,7 @@ from litellm.types.mcp import ( MCPPreCallRequestObject, MCPPreCallResponseObject, ) +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionResult from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams from litellm.utils import ( @@ -2353,6 +2354,7 @@ class ProxyLogging: call_type: CallTypesLiteral, guardrails_only: bool = False, skip_guardrails: bool = False, + endpoint_type: EndpointType = EndpointType.GENERIC, ) -> None: pass @@ -2364,6 +2366,7 @@ class ProxyLogging: call_type: CallTypesLiteral, guardrails_only: bool = False, skip_guardrails: bool = False, + endpoint_type: EndpointType = EndpointType.GENERIC, ) -> dict: pass @@ -2374,6 +2377,7 @@ class ProxyLogging: call_type: CallTypesLiteral, guardrails_only: bool = False, skip_guardrails: bool = False, + endpoint_type: EndpointType = EndpointType.GENERIC, ) -> dict | None: """ Allows users to modify/reject the incoming request to the proxy, without having to deal with parsing Request body. @@ -2512,11 +2516,21 @@ class ProxyLogging: if call_type in MCP_GUARDRAIL_CALL_TYPES and user_api_key_dict is None: continue - response: Exception | str | Mapping[str, object] | None = await _callback.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=self.call_details["user_api_key_cache"], - data=data, - call_type=call_type, + response: Exception | str | Mapping[str, object] | None = ( + await _callback.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=self.call_details["user_api_key_cache"], + data=data, + call_type=call_type, + endpoint_type=endpoint_type, + ) + if isinstance(_callback, _PROXY_MaxParallelRequestsHandler_v3) + else await _callback.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=self.call_details["user_api_key_cache"], + data=data, + call_type=call_type, + ) ) if response is not None: data = await self.process_pre_call_hook_response( diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index 619001a5791..bdfe99403a5 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -30,6 +30,7 @@ class EndpointType(str, Enum): OPENAI = "openai" TINYFISH = "tinyfish" GENERIC = "generic" + DECISIONS = "decisions" class PassthroughStandardLoggingPayload(TypedDict, total=False): diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 4577d578263..fb903043799 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7,7 +7,7 @@ import os import traceback from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping from types import MappingProxyType, SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest import mock from unittest.mock import AsyncMock, MagicMock, Mock, patch from urllib.parse import parse_qs @@ -7292,6 +7292,82 @@ class TestTypeSafePassthroughRoute: assert sent.headers["authorization"] == "Bearer typesafe-test-key" assert json.loads(sent.content or b"{}") == (body or {}) + @pytest.mark.parametrize( + "provider, endpoint, is_decision_request", + ( + ("typesafe", "systemone", True), + ("typesafe", "systemone/", True), + ("typesafe", "systemone?trace=1", True), + ("typesafe", "systemone/?trace=1", True), + ("typesafe", "systemone/other", False), + ("typesafe", "systemone/other/", False), + ("typesafe", "systemone-other", False), + ("typesafe", "chat/completions?next=/typesafe/v1/systemone", False), + ("openrouter", "systemone", False), + ("openrouter", "systemone/", False), + ("openrouter", "chat/completions", False), + ), + ) + @pytest.mark.parametrize("quota_scope", ("key", "project_output")) + @pytest.mark.parametrize("token_limit", (0, 1000)) + def test_token_limits_preserve_decisions_cap_generation_and_enforce_quota( + self, + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + provider: Literal["typesafe", "openrouter"], + endpoint: str, + is_decision_request: bool, + quota_scope: Literal["key", "project_output"], + token_limit: int, + ) -> None: + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _PROXY_MaxParallelRequestsHandler_v3, + get_request_stash, + ) + from litellm.proxy.utils import InternalUsageCache, ProxyLogging + + cache: Final = DualCache() + limiter: Final = _PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=InternalUsageCache(cache)) + monkeypatch.setattr(litellm, "callbacks", list((limiter, _PROXY_CacheControlCheck()))) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=cache)) + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.setenv("OPENROUTER_API_BASE", "https://typesafe.example/base") + model: Final = "jev-latest" if provider == "typesafe" else "test-generative-model" + auth: Final = UserAPIKeyAuth( + api_key="sk-limited", + tpm_limit=token_limit if quota_scope == "key" else None, + project_id="test-project" if quota_scope == "project_output" else None, + project_metadata={"model_otpm_limit": {model: token_limit}} if quota_scope == "project_output" else {}, + ) + monkeypatch.setitem(proxy_server.app.dependency_overrides, user_api_key_auth, lambda: auth) + body: Final = ( + { + "model": model, + "state": "A request for help", + "questions": {"urgent": {"type": "noul", "instructions": "Is this urgent?"}}, + } + if is_decision_request + else {"model": model, "messages": [{"role": "user", "content": "Hello"}]} + ) + + def upstream_response(request: httpx.Request) -> httpx.Response: + expected_body: Final = body if is_decision_request else {**body, "max_tokens": token_limit // 4} + assert json.loads(request.content) == expected_body + stash: Final = get_request_stash() + assert stash is not None + assert (stash.reserved_tokens if quota_scope == "key" else stash.otpm_reserved_tokens) > 0 + return httpx.Response(200, json={"model": model}) + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post(f"https://typesafe.example/base/v1/{endpoint}").mock(side_effect=upstream_response) + response: Final = client.post(f"/{provider}/v1/{endpoint}", json=body) + + assert response.status_code == (429 if token_limit == 0 else 200), response.text + assert route.call_count == (0 if token_limit == 0 else 1) + @pytest.mark.asyncio async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch): monkeypatch.setenv("TYPESAFE_API_KEY", "typesafe-test-key") diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 81ccc66942a..793db970dd5 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -49,6 +49,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.types import utils as types_utils from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + EndpointType, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, ) @@ -1689,7 +1690,7 @@ async def test_pass_through_request_streamed_response_is_owned_by_the_caller(): cache_dict[cache_key] = SimpleNamespace(client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) mock_proxy_logging = MagicMock() - mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type, endpoint_type: data) mock_proxy_logging.post_call_failure_hook = AsyncMock() mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) mock_proxy_logging.get_proxy_hook = MagicMock(return_value=MagicMock()) @@ -2591,7 +2592,9 @@ async def _run_pass_through_and_capture_wire_url( mock_request.body = AsyncMock(return_value=b"") mock_proxy_logging = MagicMock() - mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type, endpoint_type=None: data + ) mock_proxy_logging.post_call_failure_hook = AsyncMock() mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) mock_proxy_logging.get_proxy_hook = MagicMock(return_value=managed_files_hook) @@ -4889,7 +4892,9 @@ async def test_pass_through_request_mid_stream_upstream_drop_fires_failure_hook( cache_dict[cache_key] = SimpleNamespace(client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) mock_proxy_logging = MagicMock() - mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type, endpoint_type=None: data + ) mock_proxy_logging.post_call_failure_hook = AsyncMock() mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=None) mock_proxy_logging.get_proxy_hook = MagicMock(return_value=None) @@ -7093,7 +7098,9 @@ async def _drive_passthrough_request_and_capture_logging( captured_data: dict = {} # mutable-ok: the pre-call hook records the request data into it - async def capture_pre_call_hook(user_api_key_dict, data, call_type): + async def capture_pre_call_hook( + user_api_key_dict, data, call_type, endpoint_type: EndpointType = EndpointType.GENERIC + ): captured_data.update(data) if on_pre_call is not None: on_pre_call(data.get("litellm_logging_obj")) From 6f123b7083c71e08a03cfa9368ccecaeea98f6ab Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:23:36 -0700 Subject: [PATCH 007/171] test(proxy): migrate DB and Redis backed proxy tests into tests/integration (#43996) * test(proxy): migrate DB and Redis backed proxy tests into tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): drop a type suppression comment from the key metadata integration test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): scope integration test cleanup to owned rows and wait for backend stats flush Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): seed NULL cache_hit and bound recovery reads from below Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yuneng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_deprecated_key_lookup_cache.py | 58 +++ .../test_batch_enqueued_tokens_redis_lua.py | 63 +++ ...st_daily_activity_aggregated_breakdowns.py | 214 ++++++++++ ...test_key_metadata_recovery_probe_bounds.py | 380 ++++++++++++++++++ ...test_prompt_caching_requests_pagination.py | 286 +++++++++++++ ...st_redis_ttl_preserving_token_increment.py | 65 +++ .../test_spend_capture_rate_captured_spend.py | 56 +++ .../common_utils/test_key_rotation_e2e.py | 90 ----- .../proxy/hooks/test_batch_enqueued_tokens.py | 42 -- .../hooks/test_parallel_request_limiter_v3.py | 195 --------- .../test_common_daily_activity.py | 251 ------------ .../test_prompt_caching_requests.py | 254 +----------- .../test_key_metadata_recovery.py | 308 +------------- .../spend_tracking/test_spend_capture_rate.py | 67 --- 14 files changed, 1124 insertions(+), 1205 deletions(-) create mode 100644 tests/integration/authorization/test_deprecated_key_lookup_cache.py create mode 100644 tests/integration/spend/test_batch_enqueued_tokens_redis_lua.py create mode 100644 tests/integration/spend/test_daily_activity_aggregated_breakdowns.py create mode 100644 tests/integration/spend/test_key_metadata_recovery_probe_bounds.py create mode 100644 tests/integration/spend/test_prompt_caching_requests_pagination.py create mode 100644 tests/integration/spend/test_redis_ttl_preserving_token_increment.py create mode 100644 tests/integration/spend/test_spend_capture_rate_captured_spend.py diff --git a/tests/integration/authorization/test_deprecated_key_lookup_cache.py b/tests/integration/authorization/test_deprecated_key_lookup_cache.py new file mode 100644 index 00000000000..acd0f7bebef --- /dev/null +++ b/tests/integration/authorization/test_deprecated_key_lookup_cache.py @@ -0,0 +1,58 @@ +import os +from datetime import datetime, timedelta, timezone +from typing import Final +from uuid import uuid4 + +import pytest + +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import ( + PrismaClient, + ProxyLogging, + _deprecated_key_cache, + _lookup_deprecated_key, +) + + +@pytest.mark.asyncio +async def test_deprecated_key_grace_period_cache_hit_path() -> None: + client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) + old_token_hash: Final = f"old-{uuid4().hex}" + active_token_hash: Final = f"active-{uuid4().hex}" + _deprecated_key_cache.clear() + + await client.connect() + try: + await client.db.litellm_verificationtoken.create( + data={ + "token": active_token_hash, + "models": [], + } + ) + await client.db.litellm_deprecatedverificationtoken.create( + data={ + "token": old_token_hash, + "active_token_id": active_token_hash, + "revoke_at": datetime.now(timezone.utc) + timedelta(minutes=5), + } + ) + + first: Final = await _lookup_deprecated_key(db=client.db, hashed_token=old_token_hash) + assert first == active_token_hash + + await client.db.litellm_deprecatedverificationtoken.delete_many(where={"token": old_token_hash}) + + second: Final = await _lookup_deprecated_key(db=client.db, hashed_token=old_token_hash) + third: Final = await _lookup_deprecated_key(db=client.db, hashed_token=old_token_hash) + + assert second == active_token_hash + assert third == active_token_hash + + cached: Final = _deprecated_key_cache.get(old_token_hash) + assert isinstance(cached, tuple) + assert len(cached) == 3 + finally: + await client.db.litellm_deprecatedverificationtoken.delete_many(where={"token": old_token_hash}) + await client.db.litellm_verificationtoken.delete_many(where={"token": active_token_hash}) + _deprecated_key_cache.clear() + await client.disconnect() diff --git a/tests/integration/spend/test_batch_enqueued_tokens_redis_lua.py b/tests/integration/spend/test_batch_enqueued_tokens_redis_lua.py new file mode 100644 index 00000000000..9856550fe1f --- /dev/null +++ b/tests/integration/spend/test_batch_enqueued_tokens_redis_lua.py @@ -0,0 +1,63 @@ +import os +import uuid +from typing import Final + +import pytest +from redis import Redis + +from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCache +from litellm.proxy.hooks.batch_enqueued_tokens import ( + BatchEnqueuedTokenOverLimit, + BatchEnqueuedTokenReservation, + BatchEnqueuedTokenScope, + BatchEnqueuedTokenStore, +) +from litellm.proxy.utils import InternalUsageCache + + +@pytest.mark.asyncio +async def test_redis_lua_path_full_lifecycle() -> None: + redis_host: Final = os.environ["REDIS_HOST"] + redis_port: Final = int(os.environ["REDIS_PORT"]) + redis_cache: Final = RedisCache(host=redis_host, port=redis_port) + store: Final = BatchEnqueuedTokenStore( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=redis_cache, default_in_memory_ttl=60)) + ) + suffix: Final = uuid.uuid4().hex[:8] + key_scope: Final = BatchEnqueuedTokenScope(key="api_key", value=f"api_key-{suffix}", limit=100) + team_scope: Final = BatchEnqueuedTokenScope(key="team", value=f"team-{suffix}", limit=50) + key_counter: Final = f"batch_enqueued_tokens:api_key:api_key-{suffix}" + team_counter: Final = f"batch_enqueued_tokens:team:team-{suffix}" + batch_id: Final = f"batch_{uuid.uuid4().hex}" + record_key: Final = f"batch_enqueued_token_reservation:{batch_id}" + + try: + over: Final = await store.reserve(tokens=60, scopes=(key_scope, team_scope)) + assert over == BatchEnqueuedTokenOverLimit(scope=team_scope, enqueued=0) + + reservation: Final = await store.reserve(tokens=50, scopes=(key_scope, team_scope)) + assert isinstance(reservation, BatchEnqueuedTokenReservation) + assert reservation.backend == "redis" + with Redis(host=redis_host, port=redis_port) as raw: + assert int(raw.get(key_counter) or 0) == 50 + assert int(raw.get(team_counter) or 0) == 50 + + assert isinstance(await store.reserve(tokens=1, scopes=(key_scope, team_scope)), BatchEnqueuedTokenOverLimit) + + await store.save_reservation(batch_id, reservation) + popped: Final = await store.pop_reservation(batch_id) + assert popped == reservation + assert await store.pop_reservation(batch_id) is None + + await store.refund(popped) + with Redis(host=redis_host, port=redis_port) as raw: + assert int(raw.get(key_counter) or 0) == 0 + assert int(raw.get(team_counter) or 0) == 0 + + refill: Final = await store.reserve(tokens=50, scopes=(key_scope, team_scope)) + assert isinstance(refill, BatchEnqueuedTokenReservation) + await store.refund(refill) + finally: + with Redis(host=redis_host, port=redis_port) as raw: + raw.delete(key_counter, team_counter, record_key) diff --git a/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py b/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py new file mode 100644 index 00000000000..56c936a716f --- /dev/null +++ b/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py @@ -0,0 +1,214 @@ +import uuid +from collections.abc import Sequence +from datetime import datetime, timedelta +from typing import Final + +import pytest +from pydantic import JsonValue, TypeAdapter + +from litellm.constants import PTU_SENTINEL_API_KEY +from tests.integration._support.client import Gateway, object_value +from tests.integration._support.database import write_rows + +_URL: Final = "/user/daily/activity/aggregated" +_RESULTS: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _unique_day() -> str: + return str((datetime(1900, 1, 1) + timedelta(days=uuid.uuid4().int % 200000)).date()) + + +def _seed(day: str, rows: Sequence[tuple[object, ...]]) -> None: + for row in rows: + write_rows( + 'INSERT INTO "LiteLLM_DailyUserSpend" (id, user_id, date, api_key, model, model_group,' + " custom_llm_provider, mcp_namespaced_tool_name, endpoint, prompt_tokens, spend, api_requests," + " successful_requests, updated_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, now())", + tuple(str(value) if isinstance(value, (int, float)) else value for value in row), + ) + + +def _clean(day: str) -> None: + write_rows('DELETE FROM "LiteLLM_DailyUserSpend" WHERE date = %s', (day,)) + + +def _activity(gateway: Gateway, day: str, **params: str) -> dict[str, JsonValue]: + response: Final = gateway.request("GET", _URL, params={"start_date": day, "end_date": day, **params}) + assert response.status_code == 200, response.text + return object_value(response.json()) + + +def _row_id() -> str: + return f"agg-{uuid.uuid4().hex}" + + +@pytest.mark.asyncio +async def test_get_daily_activity_aggregated_returns_every_api_key(gateway: Gateway) -> None: + day: Final = _unique_day() + _seed( + day, + [ + *[ + ( + _row_id(), + f"user-{i:03d}", + day, + f"key-{i:03d}", + "gpt-5", + "", + "openai", + None, + "/v1/chat/completions", + 10, + 6.0 if i == 4 else float(i + 1), + 1, + 1, + ) + for i in range(105) + ], + (_row_id(), None, day, PTU_SENTINEL_API_KEY, "gpt-5", "", "azure", None, None, 0, 1000.0, 0, 0), + ], + ) + try: + body: Final = _activity(gateway, day) + metadata: Final = object_value(body["metadata"]) + assert metadata["total_spend"] == pytest.approx(6566.0) + assert metadata["total_api_requests"] == 105 + results: Final = _RESULTS.validate_python(body["results"]) + assert len(results) == 1 + result_day: Final = object_value(results[0]) + assert object_value(result_day["metrics"])["spend"] == pytest.approx(6566.0) + breakdown: Final = object_value(result_day["breakdown"]) + expected_api_keys: Final = {f"key-{i:03d}" for i in range(105)} + api_keys: Final = object_value(breakdown["api_keys"]) + assert set(api_keys) == expected_api_keys + assert PTU_SENTINEL_API_KEY not in api_keys + models: Final = object_value(breakdown["models"]) + gpt5: Final = object_value(models["gpt-5"]) + assert object_value(gpt5["metrics"])["spend"] == pytest.approx(6566.0) + assert set(object_value(gpt5["api_key_breakdown"])) == expected_api_keys + providers: Final = object_value(breakdown["providers"]) + openai: Final = object_value(providers["openai"]) + assert object_value(openai["metrics"])["spend"] == pytest.approx(5566.0) + assert set(object_value(openai["api_key_breakdown"])) == expected_api_keys + endpoints: Final = object_value(breakdown["endpoints"]) + assert object_value(object_value(endpoints["/v1/chat/completions"])["metrics"])["api_requests"] == 105 + finally: + _clean(day) + + +@pytest.mark.asyncio +async def test_get_daily_activity_aggregated_explicit_api_key_filter_scopes_results( + gateway: Gateway, +) -> None: + day: Final = _unique_day() + _seed( + day, + [ + ( + _row_id(), + f"user-{i}", + day, + f"key-{i}", + "gpt-5", + "", + "openai", + None, + "/v1/chat/completions", + 10, + float(i + 1), + 1, + 1, + ) + for i in range(3) + ], + ) + try: + body: Final = _activity(gateway, day, api_key="key-1") + assert object_value(body["metadata"])["total_spend"] == 2.0 + results: Final = _RESULTS.validate_python(body["results"]) + assert len(results) == 1 + breakdown: Final = object_value(object_value(results[0])["breakdown"]) + api_keys: Final = object_value(breakdown["api_keys"]) + assert set(api_keys) == {"key-1"} + assert object_value(object_value(api_keys["key-1"])["metrics"])["spend"] == 2.0 + gpt5: Final = object_value(object_value(breakdown["models"])["gpt-5"]) + assert object_value(gpt5["metrics"])["spend"] == 2.0 + assert set(object_value(gpt5["api_key_breakdown"])) == {"key-1"} + finally: + _clean(day) + + +@pytest.mark.asyncio +async def test_get_daily_activity_aggregated_model_group_rollups_fall_back_to_model_name( + gateway: Gateway, +) -> None: + day: Final = _unique_day() + _seed( + day, + [ + ( + _row_id(), + "user-0", + day, + "key-0", + "gpt-5", + "gpt-5-eu", + "openai", + None, + "/v1/chat/completions", + 10, + 7.0, + 1, + 1, + ), + ( + _row_id(), + "user-1", + day, + "key-1", + "gpt-5", + "", + "openai", + None, + "/v1/chat/completions", + 10, + 3.0, + 1, + 1, + ), + ( + _row_id(), + "user-2", + day, + "key-2", + "claude-x", + None, + "anthropic", + None, + "/v1/messages", + 10, + 2.0, + 1, + 1, + ), + ], + ) + try: + body: Final = _activity(gateway, day) + results: Final = _RESULTS.validate_python(body["results"]) + assert len(results) == 1 + breakdown: Final = object_value(object_value(results[0])["breakdown"]) + model_groups: Final = object_value(breakdown["model_groups"]) + assert set(model_groups) == {"gpt-5-eu", "gpt-5", "claude-x"} + assert object_value(object_value(model_groups["gpt-5-eu"])["metrics"])["spend"] == 7.0 + gpt5_group: Final = object_value(model_groups["gpt-5"]) + assert object_value(gpt5_group["metrics"])["spend"] == 3.0 + assert object_value(object_value(model_groups["claude-x"])["metrics"])["spend"] == 2.0 + assert set(object_value(gpt5_group["api_key_breakdown"])) == {"key-1"} + models: Final = object_value(breakdown["models"]) + assert set(models) == {"gpt-5", "claude-x"} + assert object_value(object_value(models["gpt-5"])["metrics"])["spend"] == 10.0 + finally: + _clean(day) diff --git a/tests/integration/spend/test_key_metadata_recovery_probe_bounds.py b/tests/integration/spend/test_key_metadata_recovery_probe_bounds.py new file mode 100644 index 00000000000..81be9e56f30 --- /dev/null +++ b/tests/integration/spend/test_key_metadata_recovery_probe_bounds.py @@ -0,0 +1,380 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta +from pathlib import Path +from typing import Final + +import litellm_proxy_extras +import psycopg +import pytest +from psycopg.types.json import Jsonb +from pydantic import JsonValue + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.spend_tracking.key_metadata_recovery import recover_key_metadata_from_spend_logs +from litellm.proxy.utils import PrismaClient, ProxyLogging, hash_token +from tests.integration._support.client import eventually +from tests.integration._support.database import scratch_database, write_rows + +_SPEND_LOGS_DDL: Final = """ + CREATE TABLE "LiteLLM_SpendLogs" ( + request_id TEXT PRIMARY KEY, + api_key TEXT NOT NULL DEFAULT '', + "startTime" TIMESTAMP(3) NOT NULL, + "user" TEXT DEFAULT '', + team_id TEXT, + metadata JSONB DEFAULT '{}' + ) +""" + +_API_KEY_START_TIME_INDEX_MIGRATION: Final = ( + Path(litellm_proxy_extras.__file__).parent + / "migrations" + / "20260823000000_add_spend_logs_api_key_starttime_index" + / "migration.sql" +) + +_STATS_SQL: Final = """ + SELECT seq_scan, idx_scan, seq_tup_read, idx_tup_fetch, n_tup_ins + FROM pg_stat_user_tables + WHERE relname = 'LiteLLM_SpendLogs' +""" + +_OTHER_BACKENDS_SQL: Final = """ + SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid() AND backend_type = 'client backend' +""" + + +@dataclass(frozen=True) +class _Settle: + previous: Mapping[str, int] | None + count: int + + +def _create_spend_logs_table(database_url: str) -> None: + write_rows(_SPEND_LOGS_DDL, (), database_url=database_url) + write_rows(_API_KEY_START_TIME_INDEX_MIGRATION.read_text(), (), database_url=database_url) + + +def _spend_log_stats(database_url: str) -> dict[str, int]: + with psycopg.connect(database_url) as connection: + row: Final = connection.execute(_STATS_SQL).fetchone() + if row is None: + return {"seq_scan": 0, "idx_scan": 0, "seq_tup_read": 0, "idx_tup_fetch": 0, "n_tup_ins": 0} + return { + "seq_scan": row[0], + "idx_scan": row[1], + "seq_tup_read": row[2], + "idx_tup_fetch": row[3], + "n_tup_ins": row[4], + } + + +def _other_client_backends(database_url: str) -> int: + with psycopg.connect(database_url) as connection: + row: Final = connection.execute(_OTHER_BACKENDS_SQL).fetchone() + return 0 if row is None else int(row[0]) + + +def _settled_stats(database_url: str, seeded_rows: int | None = None) -> dict[str, int]: + eventually( + lambda: _other_client_backends(database_url), + lambda backends: backends == 0, + seconds=60, + ) + settle = _Settle(previous=None, count=0) + + def probe() -> dict[str, int]: + nonlocal settle + current: Final = _spend_log_stats(database_url) + if current == settle.previous: + settle = _Settle(previous=current, count=settle.count + 1) + else: + settle = _Settle(previous=current, count=0) + return current + + settled: Final = eventually( + probe, + lambda stats: settle.count >= 5 and (seeded_rows is None or stats["n_tup_ins"] >= seeded_rows), + seconds=60, + ) + return settled + + +def _rows_read_since(database_url: str, baseline: Mapping[str, int]) -> int: + settled: Final = _settled_stats(database_url) + return (settled["seq_tup_read"] + settled["idx_tup_fetch"]) - (baseline["seq_tup_read"] + baseline["idx_tup_fetch"]) + + +def _insert_nameless_spend_logs(connection: psycopg.Connection[tuple[object, ...]], digest: str, rows: int) -> None: + connection.execute( + """ + INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime") + SELECT %(digest)s || '-' || g, %(digest)s, %(start)s + g * interval '1 minute' + FROM generate_series(1, %(rows)s) g + """, + {"digest": digest, "start": datetime(2026, 9, 7), "rows": rows}, + ) + + +def _named_spend_log( + digest: str, logged_at: datetime, alias: str | None, user: str | None, team: str | None = None +) -> tuple[str, str, datetime, str, str | None, Jsonb]: + return ( + f"{digest}-{logged_at.isoformat()}", + digest, + logged_at, + user or "", + team, + Jsonb({"user_api_key_alias": alias} if alias else {}), + ) + + +def _insert_spend_logs(database_url: str, rows: Sequence[tuple[str, str, datetime, str, str | None, Jsonb]]) -> None: + with psycopg.connect(database_url) as connection: + connection.cursor().executemany( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime", "user", team_id, metadata)' + " VALUES (%s, %s, %s, %s, %s, %s)", + list(rows), + ) + + +def _analyze(database_url: str, vacuum: bool) -> None: + with psycopg.connect(database_url, autocommit=True) as connection: + if vacuum: + connection.execute('VACUUM (ANALYZE) "LiteLLM_SpendLogs"') + else: + connection.execute('ANALYZE "LiteLLM_SpendLogs"') + + +async def _recover( + monkeypatch: pytest.MonkeyPatch, + database_url: str, + digests: set[str] | frozenset[str], + window: tuple[datetime, datetime], +) -> Mapping[str, JsonValue]: + monkeypatch.setenv("DATABASE_URL", database_url) + client: Final = PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + return await recover_key_metadata_from_spend_logs(client, digests, window, cache=InMemoryCache()) + finally: + await client.disconnect() + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_names_a_key_by_its_oldest_and_newest_named_rows_in_the_window( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scratch_database() as database_url: + _create_spend_logs_table(database_url) + unnamed_edges, owner_logged_late, reowned, outside_window, never_named = ( + hash_token(f"cli-session-{name}") for name in ("edges", "late", "reowned", "window", "never") + ) + _insert_spend_logs( + database_url, + ( + _named_spend_log(unnamed_edges, datetime(2026, 9, 7, 1), None, None), + _named_spend_log(unnamed_edges, datetime(2026, 9, 8), "cli-a", "alice", "team-a"), + _named_spend_log(unnamed_edges, datetime(2026, 9, 9), "cli-a", "alice", "team-a"), + _named_spend_log(unnamed_edges, datetime(2026, 9, 9, 23), None, None), + _named_spend_log(owner_logged_late, datetime(2026, 9, 7, 1), "cli-b", None), + _named_spend_log(owner_logged_late, datetime(2026, 9, 9), "cli-b", "bob"), + _named_spend_log(reowned, datetime(2026, 9, 7, 1), "cli-c", "carol"), + _named_spend_log(reowned, datetime(2026, 9, 9), "cli-c", "dave"), + _named_spend_log(outside_window, datetime(2026, 9, 6), "stale-alias", "erin"), + _named_spend_log(outside_window, datetime(2026, 9, 8), "cli-d", "erin"), + _named_spend_log(outside_window, datetime(2026, 9, 10), "later-alias", "erin"), + _named_spend_log(never_named, datetime(2026, 9, 8), None, None), + ), + ) + + result: Final = await _recover( + monkeypatch, + database_url, + {unnamed_edges, owner_logged_late, reowned, outside_window, never_named}, + (datetime(2026, 9, 7), datetime(2026, 9, 10)), + ) + + assert dict(result) == { + unnamed_edges: {"key_alias": "cli-a", "team_id": "team-a", "user_id": "alice"}, + owner_logged_late: {"key_alias": "cli-b", "team_id": None, "user_id": "bob"}, + reowned: {"key_alias": "cli-c", "team_id": None, "user_id": None}, + outside_window: {"key_alias": "cli-d", "team_id": None, "user_id": "erin"}, + } + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_reads_two_rows_per_key_however_many_the_key_logged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scratch_database() as database_url: + _create_spend_logs_table(database_url) + owners: Final[Mapping[str, str]] = {hash_token(f"cli-session-busy-{i}"): f"user-{i}" for i in range(5)} + with psycopg.connect(database_url) as connection: + for digest, owner in owners.items(): + connection.execute( + """ + INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime", "user", metadata) + SELECT %(digest)s || '-' || g, %(digest)s, %(start)s + g * interval '1 minute', %(owner)s, + jsonb_build_object('user_api_key_alias', 'cli-session-' || %(owner)s) + FROM generate_series(1, 2000) g + """, + {"digest": digest, "owner": owner, "start": datetime(2026, 9, 7)}, + ) + _analyze(database_url, vacuum=False) + baseline: Final = _settled_stats(database_url, seeded_rows=10000) + + result: Final = await _recover( + monkeypatch, database_url, frozenset(owners), (datetime(2026, 9, 7), datetime(2026, 9, 10)) + ) + + assert {digest: meta.get("user_id") for digest, meta in result.items()} == owners + rows_read: Final = _rows_read_since(database_url, baseline) + assert len(owners) <= rows_read <= 10 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_walks_a_bounded_number_of_nameless_rows_per_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scratch_database() as database_url: + _create_spend_logs_table(database_url) + named_late: Final[Mapping[str, str]] = {hash_token(f"cli-session-late-{i}"): f"user-{i}" for i in range(3)} + never_named: Final = frozenset(hash_token(f"cli-session-never-{i}") for i in range(3)) + with psycopg.connect(database_url) as connection: + for digest in (*named_late, *never_named): + _insert_nameless_spend_logs(connection, digest, 3 * SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE) + for digest, owner in named_late.items(): + connection.execute( + """ + INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime", "user", metadata) + VALUES (%(digest)s || '-newest', %(digest)s, %(logged_at)s, %(owner)s, + jsonb_build_object('user_api_key_alias', 'cli-session-' || %(owner)s)) + """, + {"digest": digest, "owner": owner, "logged_at": datetime(2026, 9, 9)}, + ) + _analyze(database_url, vacuum=False) + baseline: Final = _settled_stats(database_url) + + result: Final = await _recover( + monkeypatch, + database_url, + frozenset(named_late) | never_named, + (datetime(2026, 9, 7), datetime(2026, 9, 10)), + ) + + assert {digest: meta.get("user_id") for digest, meta in result.items()} == named_late + rows_read: Final = _rows_read_since(database_url, baseline) + assert len(frozenset(named_late) | never_named) <= rows_read <= 1800 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_reads_a_short_nameless_key_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scratch_database() as database_url: + _create_spend_logs_table(database_url) + rows_per_key: Final = SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE // 2 + never_named: Final = frozenset(hash_token(f"cli-session-short-{i}") for i in range(20)) + with psycopg.connect(database_url) as connection: + for digest in never_named: + _insert_nameless_spend_logs(connection, digest, rows_per_key) + _analyze(database_url, vacuum=True) + baseline: Final = _settled_stats(database_url) + + result: Final = await _recover( + monkeypatch, database_url, never_named, (datetime(2026, 9, 7), datetime(2026, 9, 10)) + ) + + assert dict(result) == {} + rows_read: Final = _rows_read_since(database_url, baseline) + assert len(never_named) <= rows_read <= 1000 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_bounds_a_busy_nameless_key_among_short_keys_before_any_vacuum( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scratch_database() as database_url: + _create_spend_logs_table(database_url) + busy: Final = frozenset(hash_token(f"cli-session-busy-nameless-{i}") for i in range(3)) + with psycopg.connect(database_url) as connection: + for short_key in range(200): + _insert_nameless_spend_logs( + connection, + hash_token(f"cli-session-short-{short_key}"), + SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE // 5, + ) + for digest in busy: + _insert_nameless_spend_logs(connection, digest, 30 * SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE) + _analyze(database_url, vacuum=False) + baseline: Final = _settled_stats(database_url) + + result: Final = await _recover(monkeypatch, database_url, busy, (datetime(2026, 9, 7), datetime(2026, 9, 10))) + + assert dict(result) == {} + rows_read: Final = _rows_read_since(database_url, baseline) + assert len(busy) <= rows_read <= 900 + + +@pytest.mark.asyncio +async def test_recover_key_metadata_from_spend_logs_finds_a_name_logged_where_the_oldest_probe_stopped( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scratch_database() as database_url: + _create_spend_logs_table(database_url) + start: Final = datetime(2026, 9, 7) + past_the_stop, tied_with_the_stop = (hash_token(f"cli-session-{name}") for name in ("past", "tied")) + same_millisecond: Final = tuple( + start + timedelta(minutes=SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE, microseconds=n) for n in (100, 200, 300) + ) + _insert_spend_logs( + database_url, + ( + *( + _named_spend_log(past_the_stop, start + timedelta(minutes=minute), None, None) + for minute in range(1, SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE + 20) + ), + _named_spend_log( + past_the_stop, + start + timedelta(minutes=SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE + 20), + "cli-p", + "pat", + ), + *( + _named_spend_log(past_the_stop, start + timedelta(minutes=minute), None, None) + for minute in range( + SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE + 21, + SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE + 51, + ) + ), + *( + _named_spend_log(tied_with_the_stop, start + timedelta(minutes=minute), None, None) + for minute in range(1, SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE) + ), + _named_spend_log(tied_with_the_stop, same_millisecond[0], None, None), + _named_spend_log(tied_with_the_stop, same_millisecond[1], None, None), + _named_spend_log(tied_with_the_stop, same_millisecond[2], "cli-t", "tess"), + ), + ) + + _analyze(database_url, vacuum=False) + baseline: Final = _settled_stats(database_url) + digests: Final = {past_the_stop, tied_with_the_stop} + + result: Final = await _recover( + monkeypatch, + database_url, + digests, + (start, datetime(2026, 9, 10)), + ) + + assert dict(result) == { + past_the_stop: {"key_alias": "cli-p", "team_id": None, "user_id": "pat"}, + tied_with_the_stop: {"key_alias": "cli-t", "team_id": None, "user_id": "tess"}, + } + assert len(digests) <= _rows_read_since(database_url, baseline) diff --git a/tests/integration/spend/test_prompt_caching_requests_pagination.py b/tests/integration/spend/test_prompt_caching_requests_pagination.py new file mode 100644 index 00000000000..9a516c2932c --- /dev/null +++ b/tests/integration/spend/test_prompt_caching_requests_pagination.py @@ -0,0 +1,286 @@ +import json +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm.types.management_endpoints.prompt_caching_requests import ( + PromptCachingRequestFilter, + PromptCachingRequestsResponse, +) +from tests.integration._support.client import Gateway +from tests.integration._support.database import write_rows + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_JSON_ROWS: Final = TypeAdapter(list[Mapping[str, object]]) +_URL: Final = "/cost_optimization/prompt_caching/requests" +_MARKER: Final = "litellm_gateway_injected_cache" + +_EXPECTED: Final = { + "injected": ("injected-empty", "injected-deployment"), + "hits": ("zero-fallback", "nested-read", "legacy-read", "boolean-number"), + "all": ( + "zero-fallback", + "write", + "nested-write", + "nested-read", + "nested-creation", + "legacy-read", + "injected-empty", + "injected-deployment", + "boolean-number", + ), +} + + +@dataclass(frozen=True) +class _Case: + request_id: str + metadata: Mapping[str, object] + cache_hit: str | None = None + start_time: datetime = datetime(2011, 9, 1, 12, 0, 0, 123456) + + +_CASES: Final = ( + _Case("injected-empty", {_MARKER: ""}), + _Case("injected-deployment", {_MARKER: "dep-a"}), + _Case("wrong-deployment", {_MARKER: "dep-b"}), + _Case("legacy-read", {"usage_object": {"cache_read_input_tokens": 100}}), + _Case("nested-read", {"usage_object": {"prompt_tokens_details": {"cached_tokens": 100}}}), + _Case("write", {"usage_object": {"cache_creation_input_tokens": 100}}), + _Case("nested-write", {"usage_object": {"prompt_tokens_details": {"cache_write_tokens": 100}}}), + _Case("nested-creation", {"usage_object": {"prompt_tokens_details": {"cache_creation_tokens": 100}}}), + _Case( + "top-precedence", + {"usage_object": {"cache_read_input_tokens": -2, "prompt_tokens_details": {"cached_tokens": 100}}}, + ), + _Case( + "zero-fallback", + {"usage_object": {"cache_read_input_tokens": 0, "prompt_tokens_details": {"cached_tokens": 100}}}, + ), + _Case( + "fractional-precedence", + {"usage_object": {"cache_read_input_tokens": 0.5, "prompt_tokens_details": {"cached_tokens": 100}}}, + ), + _Case("malformed-number", {"usage_object": {"cache_read_input_tokens": "100"}}), + _Case("malformed-container", {"usage_object": [100]}), + _Case("boolean-number", {"usage_object": {"cache_read_input_tokens": True}}), + _Case("boolean-marker", {_MARKER: True}), + _Case("response-cache", {_MARKER: "", "usage_object": {"cache_read_input_tokens": 100}}, "True"), + _Case("outside-before", {_MARKER: ""}, start_time=datetime(2011, 8, 31, 23, 59, 59)), + _Case( + "outside-after", {"usage_object": {"cache_read_input_tokens": 100}}, start_time=datetime(2011, 9, 2, 0, 0, 1) + ), +) + + +def _window(prefix: str) -> tuple[datetime, datetime]: + day: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[2:14], 16) % 200000) + return day, day + timedelta(days=1) + + +def _seed(prefix: str, cases: tuple[_Case, ...] = _CASES) -> None: + shift: Final = _window(prefix)[0] - datetime(2011, 9, 1) + for case in cases: + write_rows( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model,' + " model_id, custom_llm_provider, spend, metadata, cache_hit)" + " VALUES (%s, 'acompletion', %s, %s::timestamp, %s::timestamp, %s, %s, %s, %s, %s::jsonb, %s)", + ( + f"{prefix}{case.request_id}", + "test-key", + (case.start_time + shift).isoformat(), + (datetime(2011, 9, 1, 12, 0, 1) + shift).isoformat(), + "claude-sonnet-5", + "dep-a", + "anthropic", + "0.01", + json.dumps(dict(case.metadata)), + case.cache_hit, + ), + ) + + +def _clean(prefix: str) -> None: + write_rows('DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",)) + + +def _strip(prefix: str, request_id: str) -> str: + assert request_id.startswith(prefix), request_id + return request_id[len(prefix) :] + + +def _run_filter_checks( + gateway: Gateway, + filter: PromptCachingRequestFilter, + prefix: str, + key: str | None, + window: tuple[datetime, datetime], +) -> None: + expected: Final = _EXPECTED[filter] + first: Final = gateway.request( + "GET", + _URL, + params={ + "start_date": window[0].isoformat(), + "end_date": window[1].isoformat(), + "filter": filter, + "page_size": "2", + }, + key=key, + ) + assert first.status_code == 200, first.text + first_page: Final = PromptCachingRequestsResponse.model_validate_json(first.content) + assert tuple(_strip(prefix, row.request_id) for row in first_page.requests) == expected[:2] + assert first_page.has_more is (len(expected) > 2) + assert (first_page.next_cursor is not None) is first_page.has_more + if first_page.next_cursor is not None: + assert _strip(prefix, first_page.next_cursor.request_id) == expected[1] + assert first_page.next_cursor.start_time == first_page.requests[-1].start_time + next_response: Final = gateway.request( + "GET", + _URL, + params={ + "start_date": window[0].isoformat(), + "end_date": window[1].isoformat(), + "filter": filter, + "page_size": "2", + "cursor_start_time": first_page.next_cursor.start_time.astimezone( + timezone(timedelta(hours=-7)) + ).isoformat(), + "cursor_request_id": first_page.next_cursor.request_id, + }, + key=key, + ) + assert next_response.status_code == 200, next_response.text + next_page: Final = PromptCachingRequestsResponse.model_validate_json(next_response.content) + assert tuple(_strip(prefix, row.request_id) for row in next_page.requests) == expected[2:4] + assert next_page.has_more is (len(expected) > 4) + assert (next_page.next_cursor is not None) is next_page.has_more + second: Final = gateway.request( + "GET", + _URL, + params={ + "start_date": window[0].isoformat(), + "end_date": window[1].isoformat(), + "filter": filter, + "page_size": "100", + }, + key=key, + ) + assert second.status_code == 200, second.text + complete: Final = PromptCachingRequestsResponse.model_validate_json(second.content) + assert tuple(_strip(prefix, row.request_id) for row in complete.requests) == expected + assert complete.has_more is False + assert complete.next_cursor is None + assert all(row.start_time.tzinfo == timezone.utc for row in complete.requests) + payload: Final = _JSON_OBJECT.validate_json(second.content) + assert set(payload) == {"requests", "page_size", "has_more", "next_cursor"} + serialized_rows: Final = _JSON_ROWS.validate_python(payload["requests"]) + assert set(serialized_rows[0]) == { + "request_id", + "start_time", + "model", + "gateway_injected", + "cache_read_tokens", + "cache_creation_tokens", + "spend", + "net_savings", + } + by_id: Final = {_strip(prefix, row.request_id): row for row in complete.requests} + if filter == "all": + assert by_id["injected-empty"].gateway_injected is True + assert by_id["injected-empty"].net_savings is None + assert by_id["legacy-read"].gateway_injected is False + assert by_id["legacy-read"].net_savings is not None and by_id["legacy-read"].net_savings > 0 + assert by_id["write"].net_savings is not None and by_id["write"].net_savings < 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("filter", ["all", "injected", "hits"]) +@pytest.mark.parametrize("role", ["admin", "view-only"]) +async def test_request_filters_match_accounting_and_paginate_before_projection( + gateway: Gateway, filter: PromptCachingRequestFilter, role: str +) -> None: + prefix: Final = f"pc{uuid.uuid4().hex[:12]}:" + _seed(prefix) + try: + if role == "admin": + _run_filter_checks(gateway, filter, prefix, None, _window(prefix)) + else: + with gateway.scenario() as scenario: + viewer: Final = scenario.user(user_role="proxy_admin_viewer") + _run_filter_checks(gateway, filter, prefix, scenario.key(user_id=viewer), _window(prefix)) + finally: + _clean(prefix) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("delete_before_cursor", [False, True]) +async def test_cursor_keeps_remaining_requests_once_during_insertions_and_deletions( + gateway: Gateway, delete_before_cursor: bool +) -> None: + prefix: Final = f"pc{uuid.uuid4().hex[:12]}:" + cases: Final = ( + *_CASES, + _Case( + "older-cache-read", + {"usage_object": {"cache_read_input_tokens": 100}}, + start_time=datetime(2011, 9, 1, 11), + ), + ) + _seed(prefix, cases) + try: + window: Final = _window(prefix) + expected: Final = (*_EXPECTED["all"], "older-cache-read") + first: Final = gateway.request( + "GET", + _URL, + params={"start_date": window[0].isoformat(), "end_date": window[1].isoformat(), "page_size": "2"}, + ) + assert first.status_code == 200, first.text + first_page: Final = PromptCachingRequestsResponse.model_validate_json(first.content) + assert tuple(_strip(prefix, row.request_id) for row in first_page.requests) == expected[:2] + assert first_page.next_cursor is not None + write_rows( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model,' + " model_id, custom_llm_provider, spend, metadata, cache_hit)" + ' SELECT %s, call_type, api_key, %s, "endTime", model, model_id, custom_llm_provider, spend,' + ' metadata, cache_hit FROM "LiteLLM_SpendLogs" WHERE request_id = %s', + (f"{prefix}newer-request", (window[0] + timedelta(hours=13)).isoformat(), f"{prefix}{expected[0]}"), + ) + write_rows( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model,' + " model_id, custom_llm_provider, spend, metadata, cache_hit)" + ' SELECT %s, call_type, api_key, %s, "endTime", model, model_id, custom_llm_provider, spend,' + ' metadata, cache_hit FROM "LiteLLM_SpendLogs" WHERE request_id = %s', + ( + f"{prefix}zz-higher-id", + (cases[0].start_time + (window[0] - datetime(2011, 9, 1))).isoformat(), + f"{prefix}{expected[0]}", + ), + ) + if delete_before_cursor: + write_rows('DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (f"{prefix}{expected[0]}",)) + following: Final = gateway.request( + "GET", + _URL, + params={ + "start_date": window[0].isoformat(), + "end_date": window[1].isoformat(), + "page_size": "100", + "cursor_start_time": first_page.next_cursor.start_time.isoformat(), + "cursor_request_id": first_page.next_cursor.request_id, + }, + ) + assert following.status_code == 200, following.text + following_page: Final = PromptCachingRequestsResponse.model_validate_json(following.content) + assert tuple(_strip(prefix, row.request_id) for row in following_page.requests) == expected[2:] + assert following_page.has_more is False + assert following_page.next_cursor is None + finally: + _clean(prefix) diff --git a/tests/integration/spend/test_redis_ttl_preserving_token_increment.py b/tests/integration/spend/test_redis_ttl_preserving_token_increment.py new file mode 100644 index 00000000000..3f623280c17 --- /dev/null +++ b/tests/integration/spend/test_redis_ttl_preserving_token_increment.py @@ -0,0 +1,65 @@ +import os +import uuid +from typing import Final + +import pytest +from redis import Redis + +from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCache +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler, +) +from litellm.proxy.utils import InternalUsageCache +from litellm.types.caching import RedisPipelineIncrementOperation + + +@pytest.mark.asyncio +async def test_async_increment_tokens_with_ttl_preservation() -> None: + redis_host: Final = os.environ["REDIS_HOST"] + redis_port: Final = int(os.environ["REDIS_PORT"]) + redis_cache: Final = RedisCache(host=redis_host, port=redis_port) + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=redis_cache)) + ) + assert handler.token_increment_script is not None + + suffix: Final = uuid.uuid4().hex[:8] + key_with_ttl: Final = f"{{test_ttl}}:with_ttl:{suffix}" + key_without_ttl: Final = f"{{test_ttl}}:without_ttl:{suffix}" + + try: + await redis_cache.async_delete_cache(key_with_ttl) + await redis_cache.async_delete_cache(key_without_ttl) + + await handler.async_increment_tokens_with_ttl_preservation( + pipeline_operations=[ + RedisPipelineIncrementOperation(key=key_with_ttl, increment_value=10.0, ttl=60), + RedisPipelineIncrementOperation(key=key_without_ttl, increment_value=5.0, ttl=None), + ] + ) + + assert await redis_cache.async_get_cache(key_with_ttl) == 10.0 + assert await redis_cache.async_get_cache(key_without_ttl) == 5.0 + first_ttl: Final = await redis_cache.async_get_ttl(key_with_ttl) + assert first_ttl is not None and 0 < first_ttl <= 60 + assert await redis_cache.async_get_ttl(key_without_ttl) is None + + with Redis(host=redis_host, port=redis_port) as raw: + assert raw.expire(key_with_ttl, 30, xx=True) == 1 + + await handler.async_increment_tokens_with_ttl_preservation( + pipeline_operations=[ + RedisPipelineIncrementOperation(key=key_with_ttl, increment_value=15.0, ttl=60), + RedisPipelineIncrementOperation(key=key_without_ttl, increment_value=7.0, ttl=None), + ] + ) + + assert await redis_cache.async_get_cache(key_with_ttl) == 25.0 + assert await redis_cache.async_get_cache(key_without_ttl) == 12.0 + second_ttl: Final = await redis_cache.async_get_ttl(key_with_ttl) + assert second_ttl is not None and 0 < second_ttl <= 30 + assert await redis_cache.async_get_ttl(key_without_ttl) is None + finally: + await redis_cache.async_delete_cache(key_with_ttl) + await redis_cache.async_delete_cache(key_without_ttl) diff --git a/tests/integration/spend/test_spend_capture_rate_captured_spend.py b/tests/integration/spend/test_spend_capture_rate_captured_spend.py new file mode 100644 index 00000000000..17bfe5779b6 --- /dev/null +++ b/tests/integration/spend/test_spend_capture_rate_captured_spend.py @@ -0,0 +1,56 @@ +from datetime import date +from typing import Final + +import pytest + +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.spend_tracking.spend_capture_rate import captured_spend_by_day +from litellm.proxy.utils import PrismaClient, ProxyLogging +from tests.integration._support.database import scratch_database, write_rows + +_DAILY_USER_SPEND_DDL: Final = """ + CREATE TABLE "LiteLLM_DailyUserSpend" ( + id TEXT PRIMARY KEY, + date TEXT NOT NULL, + custom_llm_provider TEXT, + spend DOUBLE PRECISION DEFAULT 0 + ) +""" + + +@pytest.mark.asyncio +async def test_captured_spend_sums_only_the_openai_billed_providers_inside_the_window( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scratch_database() as database_url: + monkeypatch.setenv("DATABASE_URL", database_url) + write_rows(_DAILY_USER_SPEND_DDL, (), database_url=database_url) + for index, (day, provider, spend) in enumerate( + ( + ("2026-09-19", "openai", 1.0), + ("2026-09-20", "openai", 2.0), + ("2026-09-20", "openai", 3.0), + ("2026-09-20", "text-completion-openai", 0.5), + ("2026-09-20", "anthropic", 100.0), + ("2026-09-21", "azure", 100.0), + ("2026-09-22", "openai", 4.0), + ) + ): + write_rows( + 'INSERT INTO "LiteLLM_DailyUserSpend" (id, date, custom_llm_provider, spend) VALUES (%s, %s, %s, %s)', + (f"row-{index}", day, provider, str(spend)), + database_url=database_url, + ) + client: Final = PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + captured: Final = await captured_spend_by_day( + client, + litellm_providers=("openai", "text-completion-openai"), + start_date=date(2026, 9, 20), + end_date=date(2026, 9, 21), + ) + finally: + await client.disconnect() + + assert dict(captured) == {"2026-09-20": 5.5} diff --git a/tests/test_litellm/proxy/common_utils/test_key_rotation_e2e.py b/tests/test_litellm/proxy/common_utils/test_key_rotation_e2e.py index dd6c1637cad..4a718b6b5f3 100644 --- a/tests/test_litellm/proxy/common_utils/test_key_rotation_e2e.py +++ b/tests/test_litellm/proxy/common_utils/test_key_rotation_e2e.py @@ -10,11 +10,8 @@ Covers the critical gaps: 6. Rotation count increments correctly over multiple rotations """ -import os from datetime import datetime, timedelta, timezone -from typing import cast from unittest.mock import AsyncMock, MagicMock, patch -from uuid import uuid4 import pytest @@ -24,11 +21,6 @@ from litellm.proxy._types import ( LiteLLM_VerificationToken, ) from litellm.proxy.common_utils.key_rotation_manager import KeyRotationManager -from litellm.proxy.utils import ( - PrismaClient, - _deprecated_key_cache, - _lookup_deprecated_key, -) class TestMultiPodKeyRotation: @@ -562,85 +554,3 @@ class TestKeyRotationInitialization: assert acquire_call.kwargs.get("cronjob_id") == KEY_ROTATION_JOB_NAME assert release_call.kwargs.get("cronjob_id") == KEY_ROTATION_JOB_NAME - - -class TestDeprecatedKeyLookupDbE2E: - """DB-backed integration tests for deprecated key lookup behavior.""" - - @pytest.mark.asyncio - async def test_deprecated_key_grace_period_cache_hit_path(self): - """ - End-to-end validation against a real Prisma-backed DB: - - old key hash resolves through LiteLLM_DeprecatedVerificationToken - - repeated lookups hit the in-memory deprecated-key cache - - no ValueError/401 regression on subsequent requests - """ - database_url = os.getenv("DATABASE_URL") - if not database_url: - pytest.skip("DATABASE_URL not set; skipping DB-backed key-rotation E2E test.") - db_url = cast(str, database_url) - - proxy_logging_obj = MagicMock() - proxy_logging_obj.failure_handler = AsyncMock() - prisma_client = PrismaClient( - database_url=db_url, proxy_logging_obj=proxy_logging_obj - ) - - old_token_hash = f"old-{uuid4().hex}" - active_token_hash = f"active-{uuid4().hex}" - _deprecated_key_cache.clear() - - await prisma_client.connect() - try: - await prisma_client.db.litellm_verificationtoken.create( - data={ - "token": active_token_hash, - "models": [], - } - ) - - await prisma_client.db.litellm_deprecatedverificationtoken.create( - data={ - "token": old_token_hash, - "active_token_id": active_token_hash, - "revoke_at": datetime.now(timezone.utc) + timedelta(minutes=5), - } - ) - - # Request 1 (DB path) + Request 2/3 (cache-hit path) - r1 = await _lookup_deprecated_key( - db=prisma_client.db, - hashed_token=old_token_hash, - ) - r2 = await _lookup_deprecated_key( - db=prisma_client.db, - hashed_token=old_token_hash, - ) - r3 = await _lookup_deprecated_key( - db=prisma_client.db, - hashed_token=old_token_hash, - ) - - assert r1 == active_token_hash - assert r2 == active_token_hash - assert r3 == active_token_hash - - cached = _deprecated_key_cache.get(old_token_hash) - assert isinstance(cached, tuple) - assert len(cached) == 3 - finally: - # Best-effort cleanup for idempotent reruns. - try: - await prisma_client.db.litellm_deprecatedverificationtoken.delete_many( - where={"token": old_token_hash} - ) - except Exception: - pass - try: - await prisma_client.db.litellm_verificationtoken.delete_many( - where={"token": active_token_hash} - ) - except Exception: - pass - _deprecated_key_cache.clear() - await prisma_client.disconnect() diff --git a/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py b/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py index e3e39a87009..40d49f5ab95 100644 --- a/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py +++ b/tests/test_litellm/proxy/hooks/test_batch_enqueued_tokens.py @@ -8,7 +8,6 @@ response-shape helpers the v3 limiter's post-call hooks rely on. import base64 import logging -import socket import uuid from collections.abc import Mapping, Sequence from types import MappingProxyType, SimpleNamespace @@ -402,47 +401,6 @@ def test_batch_response_view_accepts_batch_objects_only(): assert batch_response_view("batch_1") is None -def _local_redis_port() -> int | None: - for port in (6379,): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.settimeout(0.2) - if sock.connect_ex(("127.0.0.1", port)) == 0: - return port - return None - - -@pytest.mark.asyncio -@pytest.mark.skipif(_local_redis_port() is None, reason="requires a local Redis on 6379 for the Lua script path") -async def test_redis_lua_path_full_lifecycle(): - from litellm.caching.redis_cache import RedisCache - - port = _local_redis_port() - redis_cache = RedisCache(host="127.0.0.1", port=port) - store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=redis_cache, default_in_memory_ttl=60)) - ) - key_scope = _scope(limit=100, key="api_key") - team_scope = _scope(limit=50, key="team") - - over = await store.reserve(tokens=60, scopes=(key_scope, team_scope)) - assert over == BatchEnqueuedTokenOverLimit(scope=team_scope, enqueued=0) - - reservation = await store.reserve(tokens=50, scopes=(key_scope, team_scope)) - assert isinstance(reservation, BatchEnqueuedTokenReservation) - assert isinstance(await store.reserve(tokens=1, scopes=(key_scope, team_scope)), BatchEnqueuedTokenOverLimit) - - batch_id = f"batch_{uuid.uuid4().hex}" - await store.save_reservation(batch_id, reservation) - popped = await store.pop_reservation(batch_id) - assert popped == reservation - assert await store.pop_reservation(batch_id) is None - - await store.refund(popped) - refill = await store.reserve(tokens=50, scopes=(key_scope, team_scope)) - assert isinstance(refill, BatchEnqueuedTokenReservation) - await store.refund(refill) - - class _OpenBreakerRedis: def async_register_script(self, script: str): async def refused(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object: diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index b546b9eb965..8b9439e25d2 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -4,7 +4,6 @@ Unit Tests for the max parallel request limiter v3 for the proxy import asyncio import logging -import os import sys import time from collections.abc import Iterator, Sequence @@ -1561,200 +1560,6 @@ async def test_dynamic_rate_limiting_v3(): ), "RPM limit should be enforced when dynamic mode and failures detected" -@pytest.mark.flaky(retries=3, delay=2) -@pytest.mark.asyncio -async def test_async_increment_tokens_with_ttl_preservation(): - """ - Test TTL preservation functionality for token increment operations. - - This test verifies that: - 1. Keys are created with proper TTL on first increment - 2. TTL is preserved on subsequent increments (not reset) - 3. Both TTL and non-TTL operations work correctly in the same call - - Environment variables required: - - REDIS_HOST: Redis server hostname - - REDIS_PORT: Redis server port - - REDIS_PASSWORD: Redis password (optional) - - Test scenario: - 1. First call: Create keys with TTL=60s and TTL=None - 2. Wait 2 seconds - 3. Second call: Increment same keys - 4. Verify TTL decreased but wasn't reset to 60s - """ - import time - - from litellm.caching.redis_cache import RedisCache - from litellm.types.caching import RedisPipelineIncrementOperation - - # Skip test if Redis environment variables are not set - redis_host = os.getenv("REDIS_HOST") - redis_port = os.getenv("REDIS_PORT") - redis_password = os.getenv("REDIS_PASSWORD") - - if not redis_host or not redis_port: - pytest.skip("Redis environment variables (REDIS_HOST, REDIS_PORT) not set") - - # Setup Redis cache - redis_cache = RedisCache( - host=redis_host, - port=int(redis_port), - password=redis_password, - ) - - local_cache = DualCache(redis_cache=redis_cache) - parallel_request_handler = _PROXY_MaxParallelRequestsHandler( - internal_usage_cache=InternalUsageCache(local_cache) - ) - - # Verify Redis connection is working - try: - await redis_cache.ping() - except Exception as e: - pytest.skip(f"Redis connection failed: {str(e)}") - - # Verify the TTL preservation script is registered - if parallel_request_handler.token_increment_script is None: - pytest.skip( - "Token increment script not available - Redis Lua scripting may not be supported" - ) - - # Test keys - use hash tags to ensure they map to same Redis cluster slot - # Use a unique suffix per test run to avoid stale state from prior runs - import uuid - - unique_suffix = str(uuid.uuid4())[:8] - test_key_with_ttl = f"{{test_ttl}}:with_ttl:{unique_suffix}" - test_key_without_ttl = f"{{test_ttl}}:without_ttl:{unique_suffix}" - - try: - # Clean up any existing test keys - try: - await redis_cache.async_delete_cache(test_key_with_ttl) - await redis_cache.async_delete_cache(test_key_without_ttl) - except Exception: - # Keys might not exist, ignore cleanup errors - pass - - # First increment: Create operations with mixed TTL scenarios - pipeline_operations_first = [ - RedisPipelineIncrementOperation( - key=test_key_with_ttl, increment_value=10.0, ttl=60 - ), - RedisPipelineIncrementOperation( - key=test_key_without_ttl, increment_value=5.0, ttl=None # No TTL - ), - ] - - # Execute first increment - await parallel_request_handler.async_increment_tokens_with_ttl_preservation( - pipeline_operations=pipeline_operations_first - ) - - # Small delay to ensure Redis has processed the commands - await asyncio.sleep(0.1) - - # Verify keys exist and check initial TTL - ttl_after_first = await redis_cache.async_get_ttl(test_key_with_ttl) - value_after_first_with_ttl = await redis_cache.async_get_cache( - test_key_with_ttl - ) - value_after_first_without_ttl = await redis_cache.async_get_cache( - test_key_without_ttl - ) - - assert ( - value_after_first_with_ttl == 10.0 - ), f"First increment should set value to 10.0, got {value_after_first_with_ttl}" - assert ( - value_after_first_without_ttl == 5.0 - ), "First increment should set value to 5.0" - assert ( - ttl_after_first is not None and ttl_after_first > 0 - ), "Key with TTL should have positive TTL after first increment" - assert ttl_after_first <= 60, "TTL should not exceed the set value" - - # Check TTL for key without TTL (should be None, meaning no expiry) - ttl_no_ttl_key = await redis_cache.async_get_ttl(test_key_without_ttl) - assert ( - ttl_no_ttl_key is None - ), "Key without TTL should have no expiry (None from async_get_ttl)" - - # Wait a moment to ensure TTL decreases - await asyncio.sleep(2) - - # Second increment: Same operations to test TTL preservation - pipeline_operations_second = [ - RedisPipelineIncrementOperation( - key=test_key_with_ttl, increment_value=15.0, ttl=60 # Same TTL value - ), - RedisPipelineIncrementOperation( - key=test_key_without_ttl, increment_value=7.0, ttl=None # No TTL - ), - ] - - # Execute second increment - await parallel_request_handler.async_increment_tokens_with_ttl_preservation( - pipeline_operations=pipeline_operations_second - ) - - # Small delay to ensure Redis has processed the commands - await asyncio.sleep(0.1) - - # Verify TTL preservation and value updates - ttl_after_second = await redis_cache.async_get_ttl(test_key_with_ttl) - value_after_second_with_ttl = await redis_cache.async_get_cache( - test_key_with_ttl - ) - value_after_second_without_ttl = await redis_cache.async_get_cache( - test_key_without_ttl - ) - - assert ( - value_after_second_with_ttl == 25.0 - ), "Second increment should update value to 25.0" - assert ( - value_after_second_without_ttl == 12.0 - ), "Second increment should update value to 12.0" - - # Critical test: TTL should be preserved (not reset to 60) - assert ttl_after_second is not None, "TTL should still exist" - assert ( - ttl_after_second < ttl_after_first - ), "TTL should have decreased (not been reset)" - assert ttl_after_second > 0, "TTL should still be positive" - - # TTL should not be close to the original 60 seconds (proving it wasn't reset) - assert ( - ttl_after_second < 59 - ), "TTL should be significantly less than original, proving preservation" - - # Key without TTL should still have no expiry - ttl_no_ttl_key_after_second = await redis_cache.async_get_ttl( - test_key_without_ttl - ) - assert ( - ttl_no_ttl_key_after_second is None - ), "Key without TTL should still have no expiry" - - finally: - # Clean up test keys - try: - await redis_cache.async_delete_cache(test_key_with_ttl) - await redis_cache.async_delete_cache(test_key_without_ttl) - except Exception: - # Ignore cleanup errors - pass - - # Properly close Redis connections to prevent warnings - try: - await redis_cache.disconnect() - except Exception: - # Ignore disconnect errors - pass - - @pytest.mark.asyncio async def test_async_increment_tokens_fallback_behavior(): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 52c374fe5a5..7cc5100037e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -1,16 +1,11 @@ -import re from collections.abc import Sequence from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock -import psycopg import pytest -from psycopg.rows import dict_row -from pytest_postgresql import factories -from litellm.constants import PTU_SENTINEL_API_KEY from litellm.proxy.management_endpoints.common_daily_activity import ( _adjust_dates_for_timezone, _build_aggregated_sql_query, @@ -1454,252 +1449,6 @@ async def test_get_daily_activity_aggregated_empty_result_set(): assert result.metadata.total_compression_saved_tokens == 0 -_aggregated_postgresql_proc: Final = factories.postgresql_proc() -_aggregated_postgresql: Final = factories.postgresql("_aggregated_postgresql_proc") - -_DAILY_USER_SPEND_DDL: Final = """ - CREATE TABLE "LiteLLM_DailyUserSpend" ( - id TEXT PRIMARY KEY, - user_id TEXT, - date TEXT NOT NULL, - api_key TEXT NOT NULL, - model TEXT, - model_group TEXT, - custom_llm_provider TEXT, - mcp_namespaced_tool_name TEXT, - endpoint TEXT, - prompt_tokens BIGINT DEFAULT 0, - completion_tokens BIGINT DEFAULT 0, - cache_read_input_tokens BIGINT DEFAULT 0, - cache_creation_input_tokens BIGINT DEFAULT 0, - compression_saved_tokens BIGINT DEFAULT 0, - compression_savings_spend DOUBLE PRECISION DEFAULT 0, - prompt_caching_savings_spend DOUBLE PRECISION DEFAULT 0, - gateway_injected_caching_savings_spend DOUBLE PRECISION DEFAULT 0, - autorouter_savings_spend DOUBLE PRECISION DEFAULT 0, - spend DOUBLE PRECISION DEFAULT 0, - api_requests BIGINT DEFAULT 0, - successful_requests BIGINT DEFAULT 0, - failed_requests BIGINT DEFAULT 0, - total_response_time_ms BIGINT DEFAULT 0, - timed_requests BIGINT DEFAULT 0 - ) -""" - - -def _seed_daily_user_spend(conn: psycopg.Connection, rows: Sequence[tuple[object, ...]]) -> None: - with conn.cursor() as cur: - cur.execute(_DAILY_USER_SPEND_DDL) - cur.executemany( - """ - INSERT INTO "LiteLLM_DailyUserSpend" - (id, user_id, date, api_key, model, model_group, custom_llm_provider, - endpoint, prompt_tokens, spend, api_requests, successful_requests) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) - """, - rows, - ) - conn.commit() - - -def _psycopg_query_raw(conn: psycopg.Connection, row_counts: list[int]): - """Run the proxy's $N-parameterized SQL through psycopg, recording each result size.""" - - async def query_raw(sql: str, *params: str) -> list[dict[str, object]]: - converted: Final = re.sub(r"\$(\d+)", r"%(p\1)s", sql) - with conn.cursor(row_factory=dict_row) as cur: - cur.execute( - converted, # pyright: ignore[reportArgumentType] # psycopg stubs want a literal-typed query - {f"p{i}": v for i, v in enumerate(params, start=1)}, - ) - rows: Final = cur.fetchall() - row_counts.append(len(rows)) - return rows - - return query_raw - - -@pytest.mark.asyncio -async def test_get_daily_activity_aggregated_returns_every_api_key( - _aggregated_postgresql: psycopg.Connection, -): - n_keys: Final = 105 - key_rows: Final = [ - ( - f"row-{i:03d}", - f"user-{i:03d}", - "2026-06-01", - f"key-{i:03d}", - "gpt-5", - "", - "openai", - "/v1/chat/completions", - 10, - 6.0 if i == 4 else float(i + 1), - 1, - 1, - ) - for i in range(n_keys) - ] - sentinel_row: Final = ( - "row-ptu", - None, - "2026-06-01", - PTU_SENTINEL_API_KEY, - "gpt-5", - "", - "azure", - None, - 0, - 1000.0, - 0, - 0, - ) - _seed_daily_user_spend(_aggregated_postgresql, [*key_rows, sentinel_row]) - key_spend: Final = sum(6.0 if i == 4 else float(i + 1) for i in range(n_keys)) - expected_api_keys: Final = {f"key-{i:03d}" for i in range(n_keys)} - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = _psycopg_query_raw(_aggregated_postgresql, []) - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_usertable = MagicMock() - mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) - - result = await get_daily_activity_aggregated( - prisma_client=mock_prisma, - table_name="litellm_dailyuserspend", - entity_id_field="user_id", - entity_id=None, - entity_metadata_field=None, - start_date="2026-06-01", - end_date="2026-06-01", - model=None, - api_key=None, - ) - - assert result.metadata.total_spend == pytest.approx(key_spend + 1000.0) - assert result.metadata.total_api_requests == 105 - day: Final = result.results[0] - assert day.metrics.spend == pytest.approx(key_spend + 1000.0) - assert set(day.breakdown.api_keys) == expected_api_keys - assert PTU_SENTINEL_API_KEY not in day.breakdown.api_keys - assert day.breakdown.models["gpt-5"].metrics.spend == pytest.approx(key_spend + 1000.0) - assert set(day.breakdown.models["gpt-5"].api_key_breakdown) == expected_api_keys - assert day.breakdown.providers["openai"].metrics.spend == pytest.approx(key_spend) - assert set(day.breakdown.providers["openai"].api_key_breakdown) == expected_api_keys - assert day.breakdown.endpoints["/v1/chat/completions"].metrics.api_requests == 105 - - -@pytest.mark.asyncio -async def test_get_daily_activity_aggregated_explicit_api_key_filter_scopes_results( - _aggregated_postgresql: psycopg.Connection, -): - """An explicit api_key filter must scope the results to that key alone.""" - rows: Final = [ - ( - f"row-{i}", - f"user-{i}", - "2026-06-01", - f"key-{i}", - "gpt-5", - "", - "openai", - "/v1/chat/completions", - 10, - float(i + 1), - 1, - 1, - ) - for i in range(3) - ] - _seed_daily_user_spend(_aggregated_postgresql, rows) - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = _psycopg_query_raw(_aggregated_postgresql, []) - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_usertable = MagicMock() - mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) - - result = await get_daily_activity_aggregated( - prisma_client=mock_prisma, - table_name="litellm_dailyuserspend", - entity_id_field="user_id", - entity_id=None, - entity_metadata_field=None, - start_date="2026-06-01", - end_date="2026-06-01", - model=None, - api_key="key-1", - ) - - assert result.metadata.total_spend == 2.0 - day: Final = result.results[0] - assert set(day.breakdown.api_keys) == {"key-1"} - assert day.breakdown.api_keys["key-1"].metrics.spend == 2.0 - assert day.breakdown.models["gpt-5"].metrics.spend == 2.0 - assert set(day.breakdown.models["gpt-5"].api_key_breakdown) == {"key-1"} - - -@pytest.mark.asyncio -async def test_get_daily_activity_aggregated_model_group_rollups_fall_back_to_model_name( - _aggregated_postgresql: psycopg.Connection, -): - """Rows stored with an empty or NULL model_group must land in the model_groups - breakdown under their model name instead of vanishing from the usage UI.""" - rows: Final = [ - ( - "row-0", - "user-0", - "2026-06-01", - "key-0", - "gpt-5", - "gpt-5-eu", - "openai", - "/v1/chat/completions", - 10, - 7.0, - 1, - 1, - ), - ("row-1", "user-1", "2026-06-01", "key-1", "gpt-5", "", "openai", "/v1/chat/completions", 10, 3.0, 1, 1), - ("row-2", "user-2", "2026-06-01", "key-2", "claude-x", None, "anthropic", "/v1/messages", 10, 2.0, 1, 1), - ] - _seed_daily_user_spend(_aggregated_postgresql, rows) - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = _psycopg_query_raw(_aggregated_postgresql, []) - mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_usertable = MagicMock() - mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) - - result = await get_daily_activity_aggregated( - prisma_client=mock_prisma, - table_name="litellm_dailyuserspend", - entity_id_field="user_id", - entity_id=None, - entity_metadata_field=None, - start_date="2026-06-01", - end_date="2026-06-01", - model=None, - api_key=None, - ) - - breakdown: Final = result.results[0].breakdown - assert set(breakdown.model_groups) == {"gpt-5-eu", "gpt-5", "claude-x"} - assert breakdown.model_groups["gpt-5-eu"].metrics.spend == 7.0 - assert breakdown.model_groups["gpt-5"].metrics.spend == 3.0 - assert breakdown.model_groups["claude-x"].metrics.spend == 2.0 - assert set(breakdown.model_groups["gpt-5"].api_key_breakdown) == {"key-1"} - assert set(breakdown.models) == {"gpt-5", "claude-x"} - assert breakdown.models["gpt-5"].metrics.spend == 10.0 - - def _no_spend_record(): """A rollup row for a key with no spend, where SUM() returns NULL (None).""" return SimpleNamespace( diff --git a/tests/test_litellm/proxy/management_endpoints/test_prompt_caching_requests.py b/tests/test_litellm/proxy/management_endpoints/test_prompt_caching_requests.py index 0995de6c39d..39dcc9630df 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_prompt_caching_requests.py +++ b/tests/test_litellm/proxy/management_endpoints/test_prompt_caching_requests.py @@ -1,148 +1,19 @@ -import json -from collections.abc import AsyncIterator, Mapping -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from types import SimpleNamespace +from collections.abc import Mapping from typing import Final import httpx -import psycopg import pytest -import pytest_asyncio from fastapi import FastAPI -from prisma import Prisma -from pydantic import TypeAdapter -from pytest_postgresql import factories from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.prompt_caching_requests import router -from litellm.proxy.spend_tracking.savings import ( - extract_cache_creation_tokens, - extract_cache_read_tokens, - marks_gateway_injection, -) -from litellm.types.management_endpoints.prompt_caching_requests import ( - PromptCachingRequestFilter, - PromptCachingRequestsResponse, -) pytestmark = pytest.mark.usefixtures("local_model_cost_map") -_cache_postgresql_proc: Final = factories.postgresql_proc() # pyright: ignore[reportUnknownMemberType] # third-party fixture factory has incomplete callable types -_cache_postgresql: Final = factories.postgresql("_cache_postgresql_proc") -_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) -_JSON_ROWS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _START: Final = "2026-09-01T00:00:00Z" _END: Final = "2026-09-02T00:00:00Z" _URL: Final = "/cost_optimization/prompt_caching/requests" -_MODEL: Final = "claude-sonnet-5" -_MARKER: Final = "litellm_gateway_injected_cache" -_DDL: Final = """ - CREATE TABLE "LiteLLM_SpendLogs" ( - request_id TEXT PRIMARY KEY, "startTime" TIMESTAMP, "endTime" TIMESTAMP, - model TEXT, model_id TEXT, custom_llm_provider TEXT, spend DOUBLE PRECISION, - metadata JSONB, cache_hit TEXT - ) -""" - - -@dataclass(frozen=True) -class _Case: - request_id: str - metadata: Mapping[str, object] - cache_hit: str | None = None - start_time: datetime = datetime(2026, 9, 1, 12, 0, 0, 123456) - - def matches(self, filter: PromptCachingRequestFilter) -> bool: - if self.cache_hit is not None and self.cache_hit.lower() == "true": - return False - if not datetime(2026, 9, 1) <= self.start_time <= datetime(2026, 9, 2): - return False - usage: Final = self.metadata.get("usage_object") - normalized: Final = _JSON_OBJECT.validate_python(usage) if isinstance(usage, Mapping) else None - injected: Final = marks_gateway_injection(self.metadata, "dep-a") - reads: Final = extract_cache_read_tokens(normalized) - writes: Final = extract_cache_creation_tokens(normalized) - match filter: - case "injected": - return injected - case "hits": - return reads > 0 - case "all": - return injected or reads > 0 or writes > 0 - - -_CASES: Final = ( - _Case("injected-empty", {_MARKER: ""}), - _Case("injected-deployment", {_MARKER: "dep-a"}), - _Case("wrong-deployment", {_MARKER: "dep-b"}), - _Case("legacy-read", {"usage_object": {"cache_read_input_tokens": 100}}), - _Case("nested-read", {"usage_object": {"prompt_tokens_details": {"cached_tokens": 100}}}), - _Case("write", {"usage_object": {"cache_creation_input_tokens": 100}}), - _Case("nested-write", {"usage_object": {"prompt_tokens_details": {"cache_write_tokens": 100}}}), - _Case("nested-creation", {"usage_object": {"prompt_tokens_details": {"cache_creation_tokens": 100}}}), - _Case( - "top-precedence", - {"usage_object": {"cache_read_input_tokens": -2, "prompt_tokens_details": {"cached_tokens": 100}}}, - ), - _Case( - "zero-fallback", - {"usage_object": {"cache_read_input_tokens": 0, "prompt_tokens_details": {"cached_tokens": 100}}}, - ), - _Case( - "fractional-precedence", - {"usage_object": {"cache_read_input_tokens": 0.5, "prompt_tokens_details": {"cached_tokens": 100}}}, - ), - _Case("malformed-number", {"usage_object": {"cache_read_input_tokens": "100"}}), - _Case("malformed-container", {"usage_object": [100]}), - _Case("boolean-number", {"usage_object": {"cache_read_input_tokens": True}}), - _Case("boolean-marker", {_MARKER: True}), - _Case("response-cache", {_MARKER: "", "usage_object": {"cache_read_input_tokens": 100}}, "True"), - _Case("outside-before", {_MARKER: ""}, start_time=datetime(2026, 8, 31, 23, 59, 59)), - _Case( - "outside-after", {"usage_object": {"cache_read_input_tokens": 100}}, start_time=datetime(2026, 9, 2, 0, 0, 1) - ), -) - - -@pytest_asyncio.fixture(loop_scope="function") -async def _cache_prisma( - _cache_postgresql: psycopg.Connection[tuple[object, ...]], -) -> AsyncIterator[Prisma]: - info: Final = _cache_postgresql.info - database: Final = Prisma(datasource={ - "url": f"postgresql://{info.user}@{info.host}:{info.port}/{info.dbname}?connection_limit=1", - }) - await database.connect() - try: - yield database - finally: - await database.disconnect() - - -def _seed(connection: psycopg.Connection[tuple[object, ...]], cases: tuple[_Case, ...] = _CASES) -> None: - with connection.cursor() as cursor: - cursor.execute(_DDL) - cursor.executemany( - """INSERT INTO "LiteLLM_SpendLogs" - VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s)""", - tuple( - ( - case.request_id, - case.start_time, - datetime(2026, 9, 1, 12, 0, 1), - _MODEL, - "dep-a", - "anthropic", - 0.01, - json.dumps(dict(case.metadata)), - case.cache_hit, - ) - for case in cases - ), - ) - connection.commit() def _app(role: LitellmUserRoles | None) -> FastAPI: @@ -156,79 +27,6 @@ def _app(role: LitellmUserRoles | None) -> FastAPI: return application -@pytest.mark.asyncio -@pytest.mark.parametrize("filter", ["all", "injected", "hits"]) -@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY]) -async def test_request_filters_match_accounting_and_paginate_before_projection( - _cache_postgresql: psycopg.Connection[tuple[object, ...]], - _cache_prisma: Prisma, - monkeypatch: pytest.MonkeyPatch, - filter: PromptCachingRequestFilter, - role: LitellmUserRoles, -) -> None: - from litellm.proxy import proxy_server - - _seed(_cache_postgresql) - monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=_cache_prisma)) - monkeypatch.setattr(proxy_server, "llm_router", None) - expected: Final = tuple(sorted((case.request_id for case in _CASES if case.matches(filter)), reverse=True)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=_app(role)), base_url="http://test") as client: - first: Final = await client.get( - _URL, params={"start_date": _START, "end_date": _END, "filter": filter, "page_size": 2} - ) - assert first.status_code == 200 - first_page: Final = PromptCachingRequestsResponse.model_validate_json(first.content) - assert tuple(row.request_id for row in first_page.requests) == expected[:2] - assert first_page.has_more is (len(expected) > 2) - assert (first_page.next_cursor is not None) is first_page.has_more - if first_page.next_cursor is not None: - assert first_page.next_cursor.request_id == expected[1] - assert first_page.next_cursor.start_time == first_page.requests[-1].start_time - next_response: Final = await client.get( - _URL, params={ - "start_date": _START, "end_date": _END, "filter": filter, "page_size": 2, - "cursor_start_time": first_page.next_cursor.start_time.astimezone( - timezone(timedelta(hours=-7)) - ).isoformat(), - "cursor_request_id": first_page.next_cursor.request_id, - } - ) - assert next_response.status_code == 200 - next_page: Final = PromptCachingRequestsResponse.model_validate_json(next_response.content) - assert tuple(row.request_id for row in next_page.requests) == expected[2:4] - assert next_page.has_more is (len(expected) > 4) - assert (next_page.next_cursor is not None) is next_page.has_more - second: Final = await client.get( - _URL, params={"start_date": _START, "end_date": _END, "filter": filter, "page_size": 100} - ) - assert second.status_code == 200 - complete: Final = PromptCachingRequestsResponse.model_validate_json(second.content) - assert tuple(row.request_id for row in complete.requests) == expected - assert complete.has_more is False - assert complete.next_cursor is None - assert all(row.start_time.tzinfo == timezone.utc for row in complete.requests) - payload: Final = _JSON_OBJECT.validate_json(second.content) - assert set(payload) == {"requests", "page_size", "has_more", "next_cursor"} - serialized_rows: Final = _JSON_ROWS.validate_python(payload["requests"]) - assert set(serialized_rows[0]) == { - "request_id", - "start_time", - "model", - "gateway_injected", - "cache_read_tokens", - "cache_creation_tokens", - "spend", - "net_savings", - } - by_id: Final = {row.request_id: row for row in complete.requests} - if filter == "all": - assert by_id["injected-empty"].gateway_injected is True - assert by_id["injected-empty"].net_savings is None - assert by_id["legacy-read"].gateway_injected is False - assert by_id["legacy-read"].net_savings is not None and by_id["legacy-read"].net_savings > 0 - assert by_id["write"].net_savings is not None and by_id["write"].net_savings < 0 - - @pytest.mark.asyncio @pytest.mark.parametrize("role", [None, LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) async def test_non_admin_is_denied_before_database_access( @@ -269,53 +67,3 @@ async def test_incomplete_cursor_is_rejected( ) as client: response: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END, **params}) assert response.status_code == 400 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("delete_before_cursor", [False, True]) -async def test_cursor_keeps_remaining_requests_once_during_insertions_and_deletions( - _cache_postgresql: psycopg.Connection[tuple[object, ...]], - _cache_prisma: Prisma, - monkeypatch: pytest.MonkeyPatch, - delete_before_cursor: bool, -) -> None: - from litellm.proxy import proxy_server - - cases: Final = (*_CASES, _Case( - "older-cache-read", {"usage_object": {"cache_read_input_tokens": 100}}, start_time=datetime(2026, 9, 1, 11), - )) - _seed(_cache_postgresql, cases) - monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=_cache_prisma)) - monkeypatch.setattr(proxy_server, "llm_router", None) - expected: Final = (*sorted((case.request_id for case in _CASES if case.matches("all")), reverse=True), "older-cache-read") - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=_app(LitellmUserRoles.PROXY_ADMIN)), base_url="http://test" - ) as client: - first: Final = await client.get(_URL, params={"start_date": _START, "end_date": _END, "page_size": 2}) - assert first.status_code == 200 - first_page: Final = PromptCachingRequestsResponse.model_validate_json(first.content) - assert tuple(row.request_id for row in first_page.requests) == expected[:2] - assert first_page.next_cursor is not None - with _cache_postgresql.cursor() as cursor: - cursor.executemany( - """INSERT INTO "LiteLLM_SpendLogs" - SELECT %s, %s, "endTime", model, model_id, custom_llm_provider, spend, metadata, cache_hit - FROM "LiteLLM_SpendLogs" WHERE request_id = %s""", - ( - ("newer-request", datetime(2026, 9, 1, 13), expected[0]), - ("zz-higher-id", cases[0].start_time, expected[0]), - ), - ) - if delete_before_cursor: - cursor.execute('DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (expected[0],)) - _cache_postgresql.commit() - following: Final = await client.get(_URL, params={ - "start_date": _START, "end_date": _END, "page_size": 100, - "cursor_start_time": first_page.next_cursor.start_time.isoformat(), - "cursor_request_id": first_page.next_cursor.request_id, - }) - assert following.status_code == 200 - following_page: Final = PromptCachingRequestsResponse.model_validate_json(following.content) - assert tuple(row.request_id for row in following_page.requests) == expected[2:] - assert following_page.has_more is False - assert following_page.next_cursor is None diff --git a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py index bc0bbd4dd38..7c7a0b31348 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py +++ b/tests/test_litellm/proxy/spend_tracking/test_key_metadata_recovery.py @@ -1,27 +1,19 @@ import asyncio -import re import time -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from datetime import datetime, timedelta -from pathlib import Path from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock -import litellm_proxy_extras -import psycopg import pytest from prisma.errors import PrismaError -from psycopg.rows import dict_row -from psycopg.types.json import Jsonb -from pytest_postgresql import factories from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import ( SPEND_LOG_KEY_METADATA_CACHE_TTL, SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL, SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS, - SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE, ) from litellm.proxy.spend_tracking.key_metadata_recovery import ( attach_user_details, @@ -606,304 +598,6 @@ async def test_recover_key_metadata_from_spend_logs_bounds_the_scan_with_a_state ) -_spend_logs_postgresql_proc: Final = factories.postgresql_proc() -_spend_logs_postgresql: Final = factories.postgresql("_spend_logs_postgresql_proc") - -_SPEND_LOGS_DDL: Final = """ - CREATE TABLE "LiteLLM_SpendLogs" ( - request_id TEXT PRIMARY KEY, - api_key TEXT NOT NULL DEFAULT '', - "startTime" TIMESTAMP(3) NOT NULL, - "user" TEXT DEFAULT '', - team_id TEXT, - metadata JSONB DEFAULT '{}' - ) -""" - -_API_KEY_START_TIME_INDEX_MIGRATION: Final = ( - Path(litellm_proxy_extras.__file__).parent - / "migrations" - / "20260823000000_add_spend_logs_api_key_starttime_index" - / "migration.sql" -) - -_SPEND_LOG_ROWS_READ_IN_THIS_TRANSACTION_SQL: Final = """ - SELECT COALESCE(seq_tup_read, 0) + COALESCE(idx_tup_fetch, 0) AS rows_read - FROM pg_stat_xact_user_tables - WHERE relname = 'LiteLLM_SpendLogs' -""" - - -def _create_spend_logs_table(conn: psycopg.Connection) -> None: - conn.execute(_SPEND_LOGS_DDL) # pyright: ignore[reportArgumentType] # DDL literal - conn.execute(_API_KEY_START_TIME_INDEX_MIGRATION.read_text()) # pyright: ignore[reportArgumentType] # migration file - - -def _psycopg_prisma(conn: psycopg.Connection) -> MagicMock: - async def query_raw(sql: str, *params: object) -> list[dict[str, object]]: - with conn.cursor(row_factory=dict_row) as cur: - cur.execute( - re.sub(r"\$(\d+)", r"%(p\1)s", sql), # pyright: ignore[reportArgumentType] # proxy SQL is not a literal - {f"p{i}": v for i, v in enumerate(params, start=1)}, - ) - return cur.fetchall() - - async def execute_raw(sql: str) -> int: - conn.execute(sql) # pyright: ignore[reportArgumentType] # proxy SQL is not a literal - return 0 - - mock_prisma: Final = MagicMock() - transaction: Final = MagicMock() - transaction.query_raw = AsyncMock(side_effect=query_raw) - transaction.execute_raw = AsyncMock(side_effect=execute_raw) - mock_prisma.db.tx.return_value.__aenter__.return_value = transaction - return mock_prisma - - -def _commit_and_vacuum(conn: psycopg.Connection) -> None: - conn.commit() - conn.set_autocommit(True) - conn.execute('VACUUM (ANALYZE) "LiteLLM_SpendLogs"') - conn.set_autocommit(False) - - -def _insert_nameless_spend_logs(conn: psycopg.Connection, digest: str, rows: int) -> None: - conn.execute( - """ - INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime") - SELECT %(digest)s || '-' || g, %(digest)s, %(start)s + g * interval '1 minute' - FROM generate_series(1, %(rows)s) g - """, - {"digest": digest, "start": datetime(2026, 9, 7), "rows": rows}, - ) - - -def _named_spend_log( - digest: str, logged_at: datetime, alias: str | None, user: str | None, team: str | None = None -) -> tuple[str, str, datetime, str, str | None, Jsonb]: - return ( - f"{digest}-{logged_at.isoformat()}", - digest, - logged_at, - user or "", - team, - Jsonb({"user_api_key_alias": alias} if alias else {}), - ) - - -@pytest.mark.asyncio -async def test_recover_key_metadata_from_spend_logs_names_a_key_by_its_oldest_and_newest_named_rows_in_the_window( - _spend_logs_postgresql: psycopg.Connection, -): - conn: Final = _spend_logs_postgresql - _create_spend_logs_table(conn) - unnamed_edges, owner_logged_late, reowned, outside_window, never_named = ( - hash_token(f"cli-session-{name}") for name in ("edges", "late", "reowned", "window", "never") - ) - with conn.cursor() as cur: - cur.executemany( - 'INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime", "user", team_id, metadata)' - " VALUES (%s, %s, %s, %s, %s, %s)", - ( - _named_spend_log(unnamed_edges, datetime(2026, 9, 7, 1), None, None), - _named_spend_log(unnamed_edges, datetime(2026, 9, 8), "cli-a", "alice", "team-a"), - _named_spend_log(unnamed_edges, datetime(2026, 9, 9), "cli-a", "alice", "team-a"), - _named_spend_log(unnamed_edges, datetime(2026, 9, 9, 23), None, None), - _named_spend_log(owner_logged_late, datetime(2026, 9, 7, 1), "cli-b", None), - _named_spend_log(owner_logged_late, datetime(2026, 9, 9), "cli-b", "bob"), - _named_spend_log(reowned, datetime(2026, 9, 7, 1), "cli-c", "carol"), - _named_spend_log(reowned, datetime(2026, 9, 9), "cli-c", "dave"), - _named_spend_log(outside_window, datetime(2026, 9, 6), "stale-alias", "erin"), - _named_spend_log(outside_window, datetime(2026, 9, 8), "cli-d", "erin"), - _named_spend_log(outside_window, datetime(2026, 9, 10), "later-alias", "erin"), - _named_spend_log(never_named, datetime(2026, 9, 8), None, None), - ), - ) - conn.commit() - - result = await recover_key_metadata_from_spend_logs( - _psycopg_prisma(conn), - {unnamed_edges, owner_logged_late, reowned, outside_window, never_named}, - (datetime(2026, 9, 7), datetime(2026, 9, 10)), - cache=InMemoryCache(), - ) - - assert dict(result) == { - unnamed_edges: {"key_alias": "cli-a", "team_id": "team-a", "user_id": "alice"}, - owner_logged_late: {"key_alias": "cli-b", "team_id": None, "user_id": "bob"}, - reowned: {"key_alias": "cli-c", "team_id": None, "user_id": None}, - outside_window: {"key_alias": "cli-d", "team_id": None, "user_id": "erin"}, - } - - -@pytest.mark.asyncio -async def test_recover_key_metadata_from_spend_logs_reads_two_rows_per_key_however_many_the_key_logged( - _spend_logs_postgresql: psycopg.Connection, -): - conn: Final = _spend_logs_postgresql - _create_spend_logs_table(conn) - owners: Final[Mapping[str, str]] = {hash_token(f"cli-session-busy-{i}"): f"user-{i}" for i in range(5)} - for digest, owner in owners.items(): - conn.execute( - """ - INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime", "user", metadata) - SELECT %(digest)s || '-' || g, %(digest)s, %(start)s + g * interval '1 minute', %(owner)s, - jsonb_build_object('user_api_key_alias', 'cli-session-' || %(owner)s) - FROM generate_series(1, 2000) g - """, - {"digest": digest, "owner": owner, "start": datetime(2026, 9, 7)}, - ) - conn.execute('ANALYZE "LiteLLM_SpendLogs"') - conn.commit() - - result = await recover_key_metadata_from_spend_logs( - _psycopg_prisma(conn), frozenset(owners), (datetime(2026, 9, 7), datetime(2026, 9, 10)), cache=InMemoryCache() - ) - - assert {digest: meta.get("user_id") for digest, meta in result.items()} == owners - rows_read: Final = conn.execute(_SPEND_LOG_ROWS_READ_IN_THIS_TRANSACTION_SQL).fetchone() # pyright: ignore[reportArgumentType] # SQL literal - assert rows_read is not None and rows_read[0] <= 2 * len(owners) - - -@pytest.mark.asyncio -async def test_recover_key_metadata_from_spend_logs_walks_a_bounded_number_of_nameless_rows_per_key( - _spend_logs_postgresql: psycopg.Connection, -): - conn: Final = _spend_logs_postgresql - _create_spend_logs_table(conn) - named_late: Final[Mapping[str, str]] = {hash_token(f"cli-session-late-{i}"): f"user-{i}" for i in range(3)} - never_named: Final = frozenset(hash_token(f"cli-session-never-{i}") for i in range(3)) - for digest in (*named_late, *never_named): - _insert_nameless_spend_logs(conn, digest, 3 * SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE) - for digest, owner in named_late.items(): - conn.execute( - """ - INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime", "user", metadata) - VALUES (%(digest)s || '-newest', %(digest)s, %(logged_at)s, %(owner)s, - jsonb_build_object('user_api_key_alias', 'cli-session-' || %(owner)s)) - """, - {"digest": digest, "owner": owner, "logged_at": datetime(2026, 9, 9)}, - ) - conn.execute('ANALYZE "LiteLLM_SpendLogs"') - conn.commit() - - result = await recover_key_metadata_from_spend_logs( - _psycopg_prisma(conn), - frozenset(named_late) | never_named, - (datetime(2026, 9, 7), datetime(2026, 9, 10)), - cache=InMemoryCache(), - ) - - assert {digest: meta.get("user_id") for digest, meta in result.items()} == named_late - rows_read: Final = conn.execute(_SPEND_LOG_ROWS_READ_IN_THIS_TRANSACTION_SQL).fetchone() # pyright: ignore[reportArgumentType] # SQL literal - assert rows_read is not None - assert rows_read[0] <= 3 * SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE * (len(named_late) + len(never_named)) - - -@pytest.mark.asyncio -async def test_recover_key_metadata_from_spend_logs_reads_a_short_nameless_key_once( - _spend_logs_postgresql: psycopg.Connection, -): - conn: Final = _spend_logs_postgresql - _create_spend_logs_table(conn) - rows_per_key: Final = SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE // 2 - never_named: Final = frozenset(hash_token(f"cli-session-short-{i}") for i in range(20)) - for digest in never_named: - _insert_nameless_spend_logs(conn, digest, rows_per_key) - _commit_and_vacuum(conn) - - result = await recover_key_metadata_from_spend_logs( - _psycopg_prisma(conn), never_named, (datetime(2026, 9, 7), datetime(2026, 9, 10)), cache=InMemoryCache() - ) - - assert dict(result) == {} - rows_read: Final = conn.execute(_SPEND_LOG_ROWS_READ_IN_THIS_TRANSACTION_SQL).fetchone() # pyright: ignore[reportArgumentType] # SQL literal - assert rows_read is not None - assert rows_read[0] <= rows_per_key * len(never_named) - - -@pytest.mark.asyncio -async def test_recover_key_metadata_from_spend_logs_bounds_a_busy_nameless_key_among_short_keys_before_any_vacuum( - _spend_logs_postgresql: psycopg.Connection, -): - conn: Final = _spend_logs_postgresql - _create_spend_logs_table(conn) - busy: Final = frozenset(hash_token(f"cli-session-busy-nameless-{i}") for i in range(3)) - for short_key in range(200): - _insert_nameless_spend_logs( - conn, hash_token(f"cli-session-short-{short_key}"), SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE // 5 - ) - for digest in busy: - _insert_nameless_spend_logs(conn, digest, 30 * SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE) - conn.execute('ANALYZE "LiteLLM_SpendLogs"') - conn.commit() - - result = await recover_key_metadata_from_spend_logs( - _psycopg_prisma(conn), busy, (datetime(2026, 9, 7), datetime(2026, 9, 10)), cache=InMemoryCache() - ) - - assert dict(result) == {} - rows_read: Final = conn.execute(_SPEND_LOG_ROWS_READ_IN_THIS_TRANSACTION_SQL).fetchone() # pyright: ignore[reportArgumentType] # SQL literal - assert rows_read is not None - assert rows_read[0] <= 3 * SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE * len(busy) - - -@pytest.mark.asyncio -async def test_recover_key_metadata_from_spend_logs_finds_a_name_logged_where_the_oldest_probe_stopped( - _spend_logs_postgresql: psycopg.Connection, -): - conn: Final = _spend_logs_postgresql - _create_spend_logs_table(conn) - start: Final = datetime(2026, 9, 7) - past_the_stop, tied_with_the_stop = (hash_token(f"cli-session-{name}") for name in ("past", "tied")) - same_millisecond: Final = tuple( - start + timedelta(minutes=SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE, microseconds=n) for n in (100, 200, 300) - ) - with conn.cursor() as cur: - cur.executemany( - 'INSERT INTO "LiteLLM_SpendLogs" (request_id, api_key, "startTime", "user", team_id, metadata)' - " VALUES (%s, %s, %s, %s, %s, %s)", - ( - *( - _named_spend_log(past_the_stop, start + timedelta(minutes=minute), None, None) - for minute in range(1, SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE + 20) - ), - _named_spend_log( - past_the_stop, - start + timedelta(minutes=SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE + 20), - "cli-p", - "pat", - ), - *( - _named_spend_log(past_the_stop, start + timedelta(minutes=minute), None, None) - for minute in range( - SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE + 21, SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE + 51 - ) - ), - *( - _named_spend_log(tied_with_the_stop, start + timedelta(minutes=minute), None, None) - for minute in range(1, SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE) - ), - _named_spend_log(tied_with_the_stop, same_millisecond[0], None, None), - _named_spend_log(tied_with_the_stop, same_millisecond[1], None, None), - _named_spend_log(tied_with_the_stop, same_millisecond[2], "cli-t", "tess"), - ), - ) - conn.commit() - - result = await recover_key_metadata_from_spend_logs( - _psycopg_prisma(conn), - {past_the_stop, tied_with_the_stop}, - (start, datetime(2026, 9, 10)), - cache=InMemoryCache(), - ) - - assert dict(result) == { - past_the_stop: {"key_alias": "cli-p", "team_id": None, "user_id": "pat"}, - tied_with_the_stop: {"key_alias": "cli-t", "team_id": None, "user_id": "tess"}, - } - - @pytest.mark.asyncio async def test_recover_cli_session_key_metadata_names_the_owner_only_when_the_suffix_is_a_real_user(): mock_prisma = MagicMock() diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_capture_rate.py b/tests/test_litellm/proxy/spend_tracking/test_spend_capture_rate.py index ebcdc95b8a2..cb79dd60245 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_capture_rate.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_capture_rate.py @@ -1,16 +1,12 @@ import json -import re from collections.abc import Mapping from datetime import date, datetime, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx -import psycopg import pytest -from psycopg.rows import dict_row from pydantic import ValidationError -from pytest_postgresql import factories from litellm.constants import ( SPEND_CAPTURE_RATE_CHECK_JOB_ID, @@ -22,7 +18,6 @@ from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, ProviderBillingRequestFailed, alert_message, - captured_spend_by_day, compute_capture_rate, run_scheduled_spend_capture_rate_check, run_spend_capture_rate_check, @@ -375,65 +370,3 @@ def test_settings_reject_typos_and_out_of_range_values(): json.loads('{"providers": ["openai"], "threshold": 0.8, "lookback_days": 3, "openai_project_ids": ["p"]}') ) assert (parsed.threshold, parsed.lookback_days, parsed.openai_project_ids) == (0.8, 3, ("p",)) - - -_capture_postgresql_proc: Final = factories.postgresql_proc() -_capture_postgresql: Final = factories.postgresql("_capture_postgresql_proc") - -_DAILY_USER_SPEND_DDL: Final = """ - CREATE TABLE "LiteLLM_DailyUserSpend" ( - id TEXT PRIMARY KEY, - date TEXT NOT NULL, - custom_llm_provider TEXT, - spend DOUBLE PRECISION DEFAULT 0 - ) -""" - - -class _PsycopgPrisma: - """``prisma_client.db.query_raw`` on a real connection, with ``$n`` placeholders converted for psycopg.""" - - def __init__(self, conn: psycopg.Connection) -> None: - self.db = self - self._conn = conn - - async def query_raw(self, sql: str, *params: object) -> list[dict[str, object]]: - converted: Final = re.sub(r"\$(\d+)", r"%(p\1)s", sql) - with self._conn.cursor(row_factory=dict_row) as cur: - cur.execute( - converted, # pyright: ignore[reportArgumentType] # psycopg stubs want a literal-typed query - {f"p{i}": list(v) if isinstance(v, tuple) else v for i, v in enumerate(params, start=1)}, - ) - return cur.fetchall() - - -@pytest.mark.asyncio -async def test_captured_spend_sums_only_the_openai_billed_providers_inside_the_window( - _capture_postgresql: psycopg.Connection, -): - conn: Final = _capture_postgresql - conn.execute(_DAILY_USER_SPEND_DDL) # pyright: ignore[reportArgumentType] # DDL literal - rows: Final = ( - ("2026-09-19", "openai", 1.0), - ("2026-09-20", "openai", 2.0), - ("2026-09-20", "openai", 3.0), - ("2026-09-20", "text-completion-openai", 0.5), - ("2026-09-20", "anthropic", 100.0), - ("2026-09-21", "azure", 100.0), - ("2026-09-22", "openai", 4.0), - ) - for index, (day, provider, spend) in enumerate(rows): - conn.execute( - 'INSERT INTO "LiteLLM_DailyUserSpend" (id, date, custom_llm_provider, spend) VALUES (%s, %s, %s, %s)', - (f"row-{index}", day, provider, spend), - ) - conn.commit() - - captured = await captured_spend_by_day( - _PsycopgPrisma(conn), # pyright: ignore[reportArgumentType] # duck-typed prisma for the raw query - litellm_providers=("openai", "text-completion-openai"), - start_date=date(2026, 9, 20), - end_date=date(2026, 9, 21), - ) - - assert dict(captured) == {"2026-09-20": 5.5} From d9f73245beae7cf580b853a37d5b00ce5229d0fe Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 1 Oct 2026 09:47:30 -0700 Subject: [PATCH 008/171] feat(lens): track worker spend through virtual keys (#43989) * feat(lens): bill worker analysis through virtual keys * fix(lens): pin the verified worker image and add setup proof * fix(lens): preserve network checks and redact billed analysis logs * test(lens): preserve legacy worker result submission during upgrade * fix(lens): enforce trusted worker IPs and restore coverage uploads * docs(lens): explain trusted proxy requirements for worker allowlists * fix(lens): yield to worker disconnects after the synthetic body --- .github/workflows/lens-worker.yml | 8 +- .github/workflows/test-postgres.yml | 6 +- deploy/lens/Dockerfile | 1 - deploy/lens/README.md | 14 +- deploy/lens/compose.yaml | 4 +- .../lens/screenshots/worker-billing-after.png | Bin 0 -> 60310 bytes .../screenshots/worker-billing-before.png | Bin 0 -> 55629 bytes litellm/proxy/_lazy_openapi_snapshot.json | 20 + litellm/proxy/auth/user_api_key_auth.py | 882 ++++++++++-------- litellm/proxy/engine/billing.py | 98 ++ litellm/proxy/engine/endpoints.py | 43 +- litellm/proxy/engine/inference.py | 73 +- litellm/proxy/engine/models.py | 1 + litellm/proxy/engine/repository.py | 30 +- litellm/proxy/engine/state.py | 13 +- litellm/proxy/engine/worker.py | 7 +- tests/integration/spend/test_lens_billing.py | 203 ++++ tests/proxy_behavior/lens/test_lifecycle.py | 105 ++- .../lens/worker_storage_smoke.py | 88 ++ .../unit/proxy/auth/test_user_api_key_auth.py | 24 +- tests/unit/proxy/engine/test_state.py | 42 + .../AnalysisKey.integration.test.tsx | 47 + .../lens/_components/AnalysisKey.tsx | 144 +++ .../EngineView.integration.test.tsx | 9 +- .../lens/_components/EngineView.tsx | 4 +- .../WorkerSetup.integration.test.tsx | 46 +- .../lens/_components/WorkerSetup.tsx | 83 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 61 ++ 28 files changed, 1580 insertions(+), 476 deletions(-) create mode 100644 deploy/lens/screenshots/worker-billing-after.png create mode 100644 deploy/lens/screenshots/worker-billing-before.png create mode 100644 litellm/proxy/engine/billing.py create mode 100644 tests/integration/spend/test_lens_billing.py create mode 100644 tests/proxy_behavior/lens/worker_storage_smoke.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.tsx diff --git a/.github/workflows/lens-worker.yml b/.github/workflows/lens-worker.yml index 0798425fd76..53334abaf88 100644 --- a/.github/workflows/lens-worker.yml +++ b/.github/workflows/lens-worker.yml @@ -37,7 +37,7 @@ jobs: run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} . - name: Verify standalone imports with a read-only filesystem run: | - docker run --rm --network none --read-only --cap-drop ALL \ + docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \ --security-opt no-new-privileges --entrypoint python \ lens-worker:${{ github.sha }} -c ' import os @@ -47,6 +47,12 @@ jobs: with trace_store() as store: assert store.count() == 0 ' + - name: Verify recovery after temporary storage fills + run: | + docker run --rm --network none --read-only --cap-drop ALL \ + --tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \ + -v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \ + --entrypoint python lens-worker:${{ github.sha }} /app/storage_smoke.py - name: Publish versioned Lens worker if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' env: diff --git a/.github/workflows/test-postgres.yml b/.github/workflows/test-postgres.yml index 1ffb7f67f16..ccdf6ef3558 100644 --- a/.github/workflows/test-postgres.yml +++ b/.github/workflows/test-postgres.yml @@ -135,7 +135,7 @@ jobs: env: TEST_PATH: ${{ matrix.test-path }} WORKERS: ${{ matrix.workers }} - PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=litellm/proxy/engine --cov-report=xml:coverage-lens-postgres.xml' || '' }} + PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || '' }} run: | if [ "${WORKERS}" = "0" ]; then uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 @@ -145,9 +145,11 @@ jobs: - name: Upload Lens database coverage if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled() - uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: use_oidc: true + version: v11.3.1 + root_dir: ${{ github.workspace }} files: coverage-lens-postgres.xml flags: lens-postgres fail_ci_if_error: true diff --git a/deploy/lens/Dockerfile b/deploy/lens/Dockerfile index dc6f61a4d94..360211194e2 100644 --- a/deploy/lens/Dockerfile +++ b/deploy/lens/Dockerfile @@ -2,6 +2,5 @@ FROM python:3.12-slim WORKDIR /app RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7 COPY litellm/proxy/engine/__init__.py litellm/proxy/engine/models.py litellm/proxy/engine/trace_store.py litellm/proxy/engine/analysis.py litellm/proxy/engine/worker.py /app/engine/ -VOLUME /tmp USER 65532:65532 CMD ["python", "-m", "engine.worker"] diff --git a/deploy/lens/README.md b/deploy/lens/README.md index 22754f89287..d4bcddf8613 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -6,9 +6,9 @@ Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL, agent tracing (`general_settings.tracing: {store: clickhouse}`), and ClickHouse configured through `CLICKHOUSE_URL` and a separate SELECT-only `CLICKHOUSE_READER_URL`. Enable the ClickHouse callback and request/response logging to analyze LLM requests. Lens can only inspect content you actually retain -In Lens, click **Connect worker**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Worker connected** when the container checks in +In Lens, click **Set up analysis**, choose an existing virtual key or **Create worker key**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Analyzer connected** when the container checks in -The command already contains the compatible worker image and one worker token. No separate API key, source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis +The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds @@ -20,7 +20,11 @@ docker compose --env-file /path/to/lens.env -f compose.yaml up -d Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build` -The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its configured router; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential +The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options + +The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential + +If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address V1 setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match @@ -60,7 +64,7 @@ Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable ex PostgreSQL stores configurations, findings and all scan history, returned in pages of 50 jobs. Workers claim jobs with optimistic concurrency and a five-minute lease, renewed every 30 seconds. A disconnected job can be reclaimed up to three times. Cancellation stops subsequent work; a model call already in flight may finish and incur cost -Before every model call, Lens reserves a conservative amount against the monthly lens budget. Successful calls reconcile to reported cost where pricing is available. Interrupted calls retain their reservation because the provider may have charged. A scan stops when the next reservation would exceed the limit, so it can stop with some budget remaining. Lens budgets are separate from virtual-key budgets; analysis calls use the proxy router directly +Before every model call, Lens reserves a conservative amount against the monthly lens budget. Successful calls reconcile to reported cost where pricing is available. Interrupted calls retain their reservation because the provider may have charged. A scan stops when the next reservation would exceed the limit, so it can stop with some budget remaining. Both the Lens budget and the selected virtual key’s budgets, model permissions, and rate limits apply. Analysis spend appears under that key in Virtual Keys and normal request logs, with Lens, scan, and worker IDs in request metadata. Analysis prompts and responses are redacted from spend logs; source traces and findings remain available through the administrator-only Lens API. Existing workers need a billing key assigned in **Set up analysis** before they can resume V1 requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, cache all per-execution observations across scans, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Findings should be reviewed by a person before acting on them @@ -100,6 +104,6 @@ python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \ Set `LITELLM_API_KEY` privately. This makes paid model calls. Inspect missed and unexpected per-run labels, final findings and coverage; do not equate a passing dataset with guaranteed detection on arbitrary traces -The worker uses temporary disk space for trace content while reviewing it, and removes those files after each review. Its Docker image supplies a writable temporary volume while keeping the application filesystem read-only +The worker uses temporary disk space for trace content while reviewing it, and removes those files after each review. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates diff --git a/deploy/lens/compose.yaml b/deploy/lens/compose.yaml index 773ff00113a..0af04814c1e 100644 --- a/deploy/lens/compose.yaml +++ b/deploy/lens/compose.yaml @@ -1,10 +1,12 @@ services: lens-worker: - image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:40fdb82113dd4474cb6e833cf28552487d87c8baf61693a1c3fc2863b7968c6a} + image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:c41e932eaf3e4efbcaf8cc5027c7e93021e5b2823f21cb8785cd107e37b91c9a} environment: LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container} LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI} restart: unless-stopped read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g} cap_drop: [ALL] security_opt: [no-new-privileges:true] diff --git a/deploy/lens/screenshots/worker-billing-after.png b/deploy/lens/screenshots/worker-billing-after.png new file mode 100644 index 0000000000000000000000000000000000000000..cb8b6991036a2dd069136618fcb5f7e6286a523c GIT binary patch literal 60310 zcmeFYcUV)+wD>Or7A_a@%_E$-0$4;$35qM&vUg5UZbR?q@?9zW?<&~ZP<5Ep1J-nm43`92+)lr;yZ zgzEVW;Kn5~7|kUbzyrVuYfT>qin~@gW1*^~dsL~Jr6bleEi4tIITV-1I(3{sB)FR? zr&YSS^yqg(K{nosc9Q+7&~NeSTQCFKT@f(5cj0cZ!oC(b2vWAFA4bF^#_=Mv@6Aaw zJ+3nmcxpD^oxd@N)z`Ai;AM(Pcund1F3z8!{pe0IU33Au>JuHJu#!zb#RFP!g*veJ z)1lqb5AmW#iA`y^Q$-@f(wfwyp{A=pbKG^LeW`O6jIX=g4cj4{He!ZfbDoQiV_iK+ zUbvBe32!|xpGZwQI8OKF=wXMN`Q0!>kV;_MeSMKp29LC3-sv-}*iwaR$Aszv0D9F? zt`E&Ac@uI>+i(0O!vAjC=BZVDf~*jQP$-WENJ4_*m`SL2?(b_aEs|7nT;b}irUVosfhU=qb6?Uw?7dQ+Jvu)Jfu znEH_NC4jEi4jCGv0itxV{KMfmA^+kTs_z-!0w9r-3%*jsh!WDSC&OL*!v_G6>s0uT z@%0@vf#akSD7~yQc>R-V&Li(L8UHvI4cCx~5lF+yjeQp}>>JMcFXw=x^}d5WvlGUf z1+@*#W@$$6{EvI`rbjWBWn3wqj#dwI4U>KH5A;@hSP30N=;qlsA}-e6^t>YLrBjwe zraht`SBV=*8|E0grp!B-{#vW4eSed$1L9%}G-4zvDy)9OfNTJ2L}0%8C#zw&KeK_@ zbp5z%K6Jj|>=vr8>l~0@qEq+=Hn|QdNQjTaxL;nku-6{9G;18U?Je1EFlfwHm8(C* zD5eKUN!L*F_iJ!NxMEG`+)t4H-N_u&Xz${>`HT!oycd_64)oSfb>L-)<~>j7NhP^L9s=g5*`TrXUV3ei*z~a zeEywc&Uz?pi3S}cWl10!#$}vHph7dPxo)suR*9c`JCE^<-R7L41)(n5_2WWAL9Fi^ zj%8Q+9a-u09u#Q)YHXbY&|1@A#=Fozi0uo|FjUGP8JuL@Q6@5heql%Fs>z#hPGO zvjB;B9DYB{X?bd}h&<45Rj@8FXx)28x%epbwWm+cY5ZslG+E*)0%(?X!;xG{Y)ywcB}9TS~D-mbGL+BwEpGgb@<$` zw}o^txz8Df71-TQU{s7aQ@^DaQ3^f>NQfSyAHdmtndW$xw1yfipS{UB8OyGY>VXw5 zj%jA^m$@%NGl$x<&7;^;G)U5l68>QBMAYX^{VZAA4Rq$)sAPJNz7*KSgus}Mql;+P z?*{J=VIvQD7e9BEdgXkdeqj+2t)H~a8xvZcoO`XTByJ*U{~*41ezG(+RVG_wA=KTx7I4X5c&Yr``@0h|8(FR1P#RMn@KF%=#=mf zn=eEidNVEas4E*%2XndCxfK^H*@i-zg9e^_n?Qqq=4~KuyEe+18aV!3>B}{&6ZfT4 zBuVQ}=!s*;(7a$58C(TDv_E0?&K^~E&PT!`3Gq5u8>q0twOfzhrB^6}&L(DJsNN2A zR>wWT6yqI<9L5O!mE9!&C7k=UpD8m7hHC}0bjmVFZ|S7IBqT#T6jo5_{53(qcj`ma z$BNK6ZwpwoG`;j=OiZdmWpdB!q*CUL$vJ|mqM~4pfk>f;zx)uhq?YYQ?(`i46D3Y7 z%XO!inKtz_v*vHlXi|ae5;AnFjXcR?!hI}Jf;6g+-!^gM8lVCzP}gh4oTXDY5R;Tb zdi7BI)?JYABYSic7S7VEl(!_H9QmZEcmyF`XC%xUVQ=B!C^cR=ZOuG8n^uMy>Z={a zK($=@s9DsZCePiVAhIbd%>Ze(+KBE>ROV<{EI!n_~Lo!I6>H+#ZY;)E?< z)nwWiE@f4wW&qoDMzuyuD-~shvpg@q3^n}lqCZA61yvR&U!FT$pwRirTv>VX@!^BV z5sC}}^h)dS`u)nV(KTZ!tlzZs<5@Gks) zb#?K+<8!Nl!{sy9BCT!Go>@VGARzDd;~pU5yi}O?SB; z-De`$rvbTg14n7t_mdvkXnDo*3ZQF+<0n*4LQIT;BP?0Z?UIGJbKA$R@Ys7cUSoDL z98=wO`4vo0xL*}8e0Y8B(v~K75^%uul9lB+`0%k;3Bk(7A+);oFMY$gyWR&I`u@+S zzdU_ypaTxGdR9}t0rru3fA^A_F#D}zJ;W&dly5^Uf^a4DIbr`m&lgKPZ#!^rogh(;q#JLLo5wtJtxIYqj{U0E4Kd1wWH z*p(<#ZM*qv@Y`1X$Tb@Czog={-c-gkf%wL@kP){xC^$Yuzx+R)-T)hOJp)=W6E|Q? z2NZ_R;ONQknASMGR(PA4yWo)@#}l}_j-0HVy4%a1+8k^Nm+i$IDhJlQrI(0rQa>kL z`8p1zAK0;7q0#qnv zAvS={UtO@)rcV$yptL|iOI#tVxJjaMLN7XrVvMsiW zKUpLXI#J1o<+f))#p(NMIcxU^H5CPqqBQkFo<(~Fb($BXj;Q{sbrN=LsOEF}i;%X|Ov;EBx-s=xe~K_k+m})X_1u$?49x zWN5II?BQYf13wa(s)+5+R@HDGi53@LYKm}AFmPvR%CG}9#%D)zNnc-X;)o&|0GYMuPuKKdQ=}hwx$aAdsHcHLQ_2<2)P*U(Y~cntS{cNiD$bfS_C!Uq1<=hlX1y?m{jOVXh6`uiT*?21zSStJ+V#A!Bu_| zG=zIZH+@&s=H49(O%)Y8HoxNx9D#SO(-p%nQj!_H%RM20W9xPBzS~+dp1(q*WEQf+k?l^T*_SW=0n_R1L7gox z^)6he3_vND|@zJ}&5xKHUvW`nbX4rMjL5zxugVZ-n+#TK`ogrlJuDykkii%D4 zJ;|hlc&q^mIs9T_4I&++|G^w{qMxj{C5NNnb+K z*2*JbkCU4jv_s{Jxl2wTy4fm^-4$sn_>4W3gVkl1> z=SSIY;JR)PS4)1;>SpdpmxFJ>%X|A{)A1GQEN?l=cK9^Js`~nb7z-UfYHD|EISAIL z2o_OB1j;jv*37crHiwg3D>BMBW5b|LQFcYoB!+jGd(eTe$4-1?LQ*cq?p&T=Q}kz^ zBKm!o;G1%m0v=%J$>8qB?Z@Tx4Upy@6nvrW%q z^5ZuXBcy;eAhMgQCsE0KRakt76th9hmaWJf2{Z9UcU-Z#>ot%f?`W7sXksF4{kCV4 z5^PuBYlu+-KUT{!Syy>aU%kpZ6$!8ZF2kNmW`-lzMn!`bzo!S!S{7c=DzZ4f z8&dM)mSH)k4b}Tz{j$7h-4Xj2VB6}>GfGWRPv{5o!z!}Vlw0-4L(NQB+NOePZ10ps zpbjWh%NS2j#hQ`y$<$EkdJ%lqE`#Y{$C~5VC`Qja!XZWeAZsS8H=mQI4*#rKt<}=L zjMX8I->EVJtY1F1OZi*H<~bxI-8?4WifUZ+j*(k}aKY}N#Gb@c;I`w zy?y1VEu+YL7@;35s)x2SwzlozV~!147U(5mh1JaXLkK$F9cy zlAUW(^;((Z(W|-{-n}|kxwiRiM>wmY(P4)rK0hGNa(UwN49iAe^IX&l9!8+uv&0I420-^!hINWUf%(dbe5jjwRo@Y{r9yznE3 z`k8BA`x<+t4@3yf+9?Ly3A40e?LlO{L3cGfLrGN1ebv&eCLq`NL=(t~#I7jTCt*kC zb+O6myE&u|rsRhJ84ofyz{A;>qd|)F%b_7rV;j`zd|k;z?Exo`Fp#LpBU^|TvNhFY z9Uv>nidfO(!L(R#ip;f%=69})(8I>&-V-~#c8ts+_SN~Dp!TEi=U1pMmW3aq`(!qGv$_z8v zDt$19h1vt#vM$f}=!X;o;}T^E1jDD{1vwnT<0)2x)caKKD9Xqqda+ZNtk_} z4IyT(89XuN!&mgF?INOL0IYz9v2(L2WbOM8yt0UgSjetOE2gPaMh_0u#mtZ#fHBHl zHg`39E24CSJ1k5y_VL^Y_m#XQ(_in2M8Db&)tV+@%W#m8eI|U-Vq{}-&x&!rj><)M z*{4L(!l1hf{Ll!c8}QGs<9Qh>)~v^3Rxc_f zIb_p1Yh>Kvc6(FbSTPIztYDzLjDoTA?BX#iuQ0dVr7geh7NUWL-|ir)ZI_%4_(VC% z#6(I)qz4{ufN6jz?Vp*nVM_6=SlAC*!{zKVL@8EBP+p=~)l7SX411I_aJND!MVJn& z@7h_Zj|dz(d;aO?eE~7cYlxO@^&0t~D;BV#Q`_0@+p2D_)%3T2xCBG7-$e9_A4qyu zW(83e{lUAQeg@}N#WcsEU@APnf!XyMD6-Cs*WaHxmhyvr`-F$&on>^xIHK+I2Ffua zv+2(pJjoH6E6|VRAmBC{)}yEZ9>Od?21l*7OEXPA{6-u@@p3C)ELSh9E3F z!0r$iWeErTDh7YK^ieKWsdXpg93W5_1S~dCUi*=9i%Z_k@>*KW;%_O>T-ap@Qpf*u zJ^lyX3O?l8b4}$CX@+V{R)ZvhEO~D`&7No=v(U5k6z-wwn)?s@8TzyK{fHfNFcjG) zrJAxo$2>?6BsO%jz#{@wpXg^PR-`T0F2nmCzAE7TCVO2yFE&HD!;mvMjTcfF1hS(< z`la^(*`Ih|Pa|=`i_5A-gptT}jijTWsXLicXV*D$QbKKLMH~?VW7!c;{0&%kWrxK= z)aAwY)bwK!2;x}QA}wC>>nvsdBLD`3-ahX-T88E2aQ1;<**YbWI{Zpx6axvYEZ2FzJ| z8>3l4;JtkC@lEa9qh;rSU8DmOH=*_@si2YRyKZElZhY~xYvjH%759er0Jw|WtoN26QVZ`}Dn|II`M5pSvfG^aVDVDB7& z1P?VAG~R_Qjb~;PtREQhksT7&|lk+c@IHqoI#Rx!=l+J9X}>U8}AdO^!fFnn~5@B zgB#XQZ{W)Q~+HTZ-X z*ziA|97iktOs}b-N2{k&HHNBl5KbIptOy+21R@&T8F(UbLa05hc*!ev*+ zi@)#Tpgs}v1w_HhAYSOXI$h9+krjy$!s`FXWF>y8l+{Sz<65fUhh?r1TsxDI3Q?9k z`-xzG3Lw*lG{XPeE;XZw73>O$)`OVnPYI&nc4qv7aIX{cw_&P#z$`7kWGIDc@KvC2 zzIT6PbmZCd$=*lSSMn;I7@ze%(yX{&ALU;)0RI3mWKoT)fJQ9XQy2v-jzun;jebK(V_dU1xbjr1;xJfJPB_uSfKW-&I*E5dL=dGJu`vFyA z70OzYb4F$J9s|LJ@T^0^@FtSE4)kAaVR`w}x3zRC{2zbN|Bs8ng*dB&59DmVJ6N@!NDHzUo9n;w^|oo~EBl~R zZIsfykjZ&#v!`!WeL`#>OFT)p5OPuJyM;TnJqJvg2PckwD!o6flp5FTkNYrQ@w)nXX#64$~^{uR&>k}2L+97HKhUq0QR)0LO4X86d2Mp|l^pPHVp966F62H|3f-OqU z0e>1xyyk6sbQ(^1i$W)_4h$UMk~9q|KL@1D1t%a3ne$EzWvPDSM?<~rgRt4X{^O2p zuzNDyU(W$EHZ2c~{-9OqBj!BUOcX!9C55rZ@lKF=Nrd$1&yJBRSy5NQjwR3DqyjsP zzT?RpcneaSJy2->`eG))n9Xqyb9_tXfEwg_DsX%bpk+WulxHO~-m^XUAoXO={=xmV zkWk#exGbpNwm1j;po*&uc>b?`qNY9{+Rn)GrnGk$jad8fG|4{PdH5dJ(5s?6witLl%k!kLH45{yI^H1ATHaDG=q~B^I+&0*4qK$j@t~lBn zS9cE3sXyf{ytom+G8m#ZxNMDEFUxj6(vEvHE^`j3s2};6_4bcS9=zL^H%4q9;K}N< zt)90iKl&r~cSy}K?Ced7ruI1g9DpI4zyL!!2k8Dmd0~=D#S(UWOZtG??B13Lxm$_! z3+lhcrpgP#{#F|_%>0Mg+bKlUgDn@y-@N%3@^ny=*ZoKbbV>aW zrL-n$GZL9{Yi2C&Y~l|KZc;jl4|@H(%-D>MokZ^aD!AEUJo4)gnT#vZ?D{<`N(;z{ zKd@B4$4rwIvKG^A+|B{p$~UN)vUyyIYc6DC{8CxoihDRtPR^hH(G;`vA3KXDXX4t< z0k`T;xs|O-M9DSsn8UpCZa-`Itv1AOjF>+DgA~s_MKXIaZSHWUqe?NFx@Z65{+j29 z@mb;@ZDf`IjWqk;2Z0m@^H0AqU@~)|iZim~98gUb;_p%SzKPoA!NF;ErQVd|y<@Wd zl$B^pL(Ic9$;j_KP!x&{s=DoKp#|!Y$u~B*^<_wVj)W)CK9>l z@N=Kc<1v|yUkL~q=JI`GbJ0^B%}eUp0x#a}H% zd}A6YTET6m1^U=J^}c;V@Yp|jK-Ygq`HH1_S-a)qqd84xQ{OQ=d`q%o@>pepMKs!h z_luR&IiRZo?JM*{#3*53)JI~EJ?t!U8}|{p)jY5qv_Dt(8NFh*0flZUV}5(|X5!s< zo;5Y_&h;8r&Q8S|LPIEy-V?8(Lnb4YTb2`dY=+yxM?AR>d_V69moyTEyOu!>QRM~M z9sv^x30pQNmz%?8Y|w$0hu1~#kOE%2Hqhc%S1hQQL@#XH%gtW1sJlp6SE{lKpVi{% zuk|1!U#AJr_Z@0o1G5xPdeWGHlw;C-DnTEj&GRV_YFqZBcOKVVIRj2qCS^a9w&DJw z>L0|+!f9Bi=CwY2M1St8l zKFGYh@pvNA? z-oK5GnUOX3Qk|cGq1ivHvc?X*Y;@c}RM!9j**E?dS^x?z!n0ML1dwpspPdN*-Ezsq z`00f;I@kMsnBRQ87*8hFZ3S<6-ERkaiB84*LM!{*qUB0*ld}bdgcMb9QD^n=c%rzE zf?8h+7WRC`{E|!^T=Vk4M`5ab#xJn=dmL{3K1oXNk8c9^crM8NsnM5OVb+dPcVx)azlhqJ7bsPo8E^klY|f*92tO4vO;NT!B!q z0B=4Hdc&de_={e3?yBhhKFh}+p=40;J;8o*m3)&-2J@$#HOFr&1d|bSh}krL9=gZ?$4%FXe6*Q{n8i1TW%GTksfL$D%+OWbxG-Wx zC@S1dyl$cYCMc8Jkz01EqqLBBnhud)G~|a$Mhp2$@S>|vX-2d|9%@o+IRD@>!gJN( z*#q(g$6LRM{ZjE0_Qn55Z7uo1C@hJ2Dh9WNz$}J-#mwcTi}4f52^tnNgc(Qt71gc( zC2#?dA6>g2#aVs03q5&6mrgRPPgQVlf2djSBDSq{f&FVeoqp`<>no*ciNiOxJNUHC z{8r_DNw>aijutdVYhZL%U&Pn+bCykVV1Bmsrgd1PDt@^z{^eGt9OiLED$bpuq3a|B zTVy<3g&jED(oBEExa1_!TGoH&u6!=iIOrN@RCsxICcE8cfSOFZkGW%%4 z8x&1SAUJdTUV}B~+xSBlP%BR>=ncgkeHgV5{5G73ZprCt(GLiQWu%1@!`)gO1m&-R ze6B7{eQ+wvHgfBuIj4?$m+ZmM#V9Fs?}3G^Ci;bX{n?IfrumLPsr8xM{4J#1WRx`c z5_ojS%Tb`NSyI!azTDkWAsx|Qkh6RjTNbzWBbXuZKyg^$SY=c$-7psYf=DRKY|EYS zN%7_rjAC+4(NixS9SFXc+CLYRFgvl#!dzV@T5@1UeK9%3(RBF4%f&`+JRROIH)ziN zChJ>Ap51{0JLQsNtnjheVaqn9&yT8nca3Jg+AqbhBw}AzYm-^5(GTf!0QXFRSN1z+ zM#K7;O^p#vU&*K!3W(btj;GCR$<>HlqDx#cvKPwt}NXoqq9j z=vU#~c|n?ubrG4|T7<$(_*s92-)w^o4(ugSFBmJ=x>R@3(s6%G5ZOP%FhW>5-qW%rz?`Sph-!T95a$)QBycDcC31;r&cqv#@dhMr*uxUt?~t{QPf zzr8s&wJ@nsosO|^Hc1x87jHPV8pw6%e?Bw|q^8xY$aUN1_1XSq=3Znz4qp;X0a4N| zFncC`oA#1bda&o3`#hklEZ#BIyC*xeg_hA`7aM*Imh#{ z7UKlxoy@x22~_6XhF}y+MuWFZYsEl_4vwS){8~LMw=^tFsP`CoQw&$|s!?G(Z1JBr z8?MCY8N@KZf|`J<8;?waO)ed4Pk5`)e|1a;&C5avhs^ZF=DHY+If;H<0p@iM@YYB{ zxaj#Oq4E4eS&<#hxANV7tz~*+RGSe~@Ydq|*8Bv0S?t&2mQf2Z^jPpK$!Eq{5SWHe zDuEl+g`0tOFt&g=tOu`eP&zJ`nB#msCYNr!wC(s>y6LWN?S!kGD>PF9EURv= zftD*N+^^|Zu1d=G4DMLuJE^0c+n_rT1kxi;5^6EUx=E&pN*9y@jsJ1!M7Z{PJFijx zcb@o;r0V_|;T99~=SSY#PFO-2!5je!jLEv^&pr8(aF*sqJ6iHHYu%!{8WY^<>+Lvi zW~QG+9778&C8TS$VPs}NB@0(`+sl+Eq%EnJ212-TXoD6^u3-}{-<+7j;Ju*_Dq*#s z!GSu0Y_r_goPH+8{(SSO5l@62hD7FeKx%NRJKw?_DI*vG!r~5H0C&>aDO)h6 zPv_?6x|V#yLabkz6K|o0QOdZWVj$cX1M)n^<^4E#+&I}68;YCotj88Z(PYO_Iz$yn zO2S&;3oEXT$5?WQ<2!Qd*=qQUlXB(i`z4|h-2HV_bh;QB#xRspxsW7%{!e_}{MKAe zT;_HU|6Vu#YnMw=$C_lz)1fsZpPR!<^QL?3^D+9fOprSXU8a(LLA|Z%ES-t%ofTPx zQPP>DU5Vq23mOZl>GawfUKoa&zn%DAvK*Hw)yOjo5=Lj`z*`~GnL1HK!zI%nbQ>FW z@zWSM;iz9X%y!?fdcsOVr?7LE?!&&&a*b4g=8fs+Cr4;3ysFfdO_R31Dkg=)pxKdc zd2Of+S(w=F?0{_s04@#lo2q}t;E#>SQP>mARBz5Xz+LCeXvHcKr8e6NguRN|cw$oT zB1r4;j%J4?^j|B@|Ap!t;AWfv(l$yz@jJKcWyzAflMf;6u?{}&pyd>WO#bPnJ+go<`nE;+`-&a75$x%>op`2(nLU7Itk$KVtW z9rW;oGX9RBs#*aHj+S2P?{cn(+oHw=`8FGn%~78d(}Rqi*}+wtjrGM0Z;5O=5>4jN z-~ovWRNLkwE#SE*{(JWkSkpy3xoVPo0y^&}k5JOFth+OThtJMv(BbyDDN__hxISb{ zh&kSgv$~r2;BAdri^@8<7QYQQpR*iC@fD+3IawbK2p61q(bdjh3LlsI_F1GXmk++f zAc0hT3;AY;NND$1eN>*29ORy%#AL4CeP{F=cO%2i2^RM01#WjkiEcYexJvYc!3j6% zq`Id!MI*g`L41@RR!2WkG@~@&H)!yl6A;K~8NzG5;8{7T*%360Dsy((7UCbRLFs7Z z8#FbQY45P67=Q%tII?+6x|bwxeZnmXLrj&!MCF0#YZDme?Pd>RKTss(%k z`sectpKGcN<`vTnWxl1>7?krhXvxM#iFaFB(HNHHuol#3k!G`ImvF%MuHEr+m zqERFF7_@*s!d_W249(RnKK|8qwfeVg3~m9|hK$x1S`U~RS?+78Z{u!=!vL?#dlwNxT#dh4dlSd0ZR-N4sWZwCfsM-hD8;}4 zL-oAg#6H9!Ogz6Ih%_tMT~%!;N$f;qt8Wf0~cPPXnqnx?G{^A#N=6&F#dJ& zn>!1}gt(Je?Wj_PiE$AuFx z2DS+AIr~TA=&Bxfs9DKX;r0x$l|?;5U+Ak=JJb#1-B@!aOJ=-q4BjS<`kWFi7f~B6 zC*Y1J$h+icuFjxa%2&dm&V9$r7YjG)p4yqxil_C-sV&Ex_Va1S^-?)n5%vH?{1>8;1yr^!OGNg5D zdvt)9JoSSpX-ZP}+A<_p2h!Mt%5Y%bCP3%;u+Fo{(jK<`rKg+bYQ^)!kQ6&dW@g)O z*v}YumI#~_uBbt7nZ@o6tAlb-M5S5FRbl7O!yUhb+Dyp8m>AmPGfIIxUztHlUEmVA z7xs&h*`?+lHA7cYzP$)GEj~t>8v&sy!%;+$`SY_cK3$V z<~n34WZD_6FF>JPTFiSX1;F>kggCOhfV; z1#%B!`MfOhe*NZXqw@YZw$WI++KutYAd;(vcZy&bQ;~2ZC~eqW0%^=2BCPU8zLeBR zxOW%t&4;h8s)gnX7%*eO^)hM4INZ+3L2xBsUBFFfHtoDIu-SW7z>7}E?!{Hrs-iiL zF}Gys@pQsAzT9J&h3te)5!EFw^bscqcBbi7O$J9cOkpQNpj&MYZ&n0A-XB*ELEDzu z%>9!f;6H$oLA5d>f<@#-OpB^s$GsoD6_X8g|o1@K3Nlp{k;uE^0%AWQH0H9 z!M2(y7&b%uW_AsRaju1$8kK0pf&M!U8IBX~7!8`_$yBb{!A2ob|45v>u(PPq*d`*$ z(F|CV`TXPt0=qBh$zQ}Zsj+A-JFQYrl1`YfmoILdDEr0_KWg+Y1*>~nE^`cd z2PYwUzA7=e{kFh^-x0Y@L=i9e&d9Ink67Wc zOYiEJkg7@Fc9bn15o=|$A`LOh#R=dpml&Vr!qto_#wfqn`dBnMyf3}a0-{&+pWB}k zD2@`8cm=BVXf|>PElJ*a#8%}|S1>!^25$SQwEtFG(UeuIToXDW4U$%+v#1t!fFWGi zLJ$ynSGG>|Yh<+WEKmUb3aRC%*$GNP4pEk$O{(rxItU_u;fW1eF9=K;*4hc=BqkZ|J|) z!iKQje4j{{^G`H)^Geu@e+56)r4*-8FP7Rk&R7?4n$^x+A3qASk5(sM0#uVD1MM3Z zN4HtD#KfrnKWM4}rQ4()q4K;Q6|}LkJn4r^^;G33-NTKUP$+CW3pc`5WkUu zewD!!8^wgWEw*!b(>$$@LfJ%iu8XkkAg^RQsXAO7!mX)M!?)bX8BpLkH+G_I?iUZ7 zDGO3lTZ3p>=Q>xZ4$*3 z<;~P%go(h^3EmbXuS(Zf%&&V4+HREBVNmmtQg&di34Kqj7;`c2ec*Y!sw7q;7p6%N(>olVLZR zehn94v8s7jiJ7vl(%>HKg{MvsQ$gvox!F%1ZJl2x51)|kSaJ)Jz@ym7W8=d>&T`B^ zBJ9j0`^4w{a^PLSLF-?hwHXfr?fl)LBc`Rmg_gk`2enaRpFMQIENxin=G&@to%H_2 zoA#1b%I@p0raqf}Cv3Z%1HLzA4zVdu`m#q}(F)DEeWK{5vF?$NA08 zNk;U}aPw|bW^?Y9uTpynS+abqJ~AmYP9leFGCKy`OeXr7Pr5`B=C7-j&-7Wy!s-b0 z4v+dpPytD-4qKT`qf)j>>S3cvgA#AWXaPn~0~da1 zm?iNmFJE6tS9OA)xVPm^qj71zF#>LO7IDWwa11FYvyxy1F4;cDVMty(OHt?{Z}?tI z%fP)mHG)tKD*<~=$U!2(LaD1CmOk#Mw?H7ETCtuEjzuMNpE1{J_?>5yX8QX<{Jx(o ztitWEx(6mCUz&A${;L;*p! zPQ^W7R-L*!kD78yHZ8l+yqfVKmV$zc`RbIAwS4Wb78(3@)}EGVWmS~A&(2ePl8eM_ zievpJucoGQP12{%7?sy1Hp=Q~JKju(x> z!Ys!{ue)qbhPv2-;2nh0Z7&6ya(Q9?HEQ(fNGhNAsfqZ;_wlaMFlV8|=rk#!K@w5K zoGZyLqg|^6>>YfP@9!Gy=xztxLj`!$;OOP_hiP@{JP@e+kn~Kg?%Nityy3gty&u2+ z;z^D!R2`04)#C?xWF(lO`Q>H9!;-CR)v?bFXOO=@(HMSJbjB72ROz0S=&BNYqxb|# zsenW!CC!`F6EbD#GTbkx51a!UnC_Hoh%E|K6C~?A6S{-m)OZr)tg)WbAks;>z=F>K zzd_8?{Odv>5y6r)ExRw2RVm}f}|CNV4st3~W< zwXn=gmRp7-UxpN!+J#7gQ4EtOe4SseCJaBIvoM@Pl~maD#_Rp_G<7x=lqo7#YY0m? z(eavV{FQ(6%}u?0>q67wLslo3`8m7rRGGE!zZR9xs5cl&>DgKHLE0ZG4M=h`7ANN9 zMipDko1Nn+F+n$0Ob53Xtqvml&}iCeVnROHol()CH0fc<$))y zxrW}PM+*UD-*()b+EDT$=*vQ=2kc66I%U9ocR{h%aLqMcx1PuO{7UD5)3!@y#cpG* z_2axOGu2pxAt^inNaM&aRgmo+cFG1|kUEJ7buO}Ktb-;I`}@&J*ducMV&?z-g;br! zXql4;Cv@GQzBybWsMu@u+9Pg!No{$iNTl8A+0bVOm|F1$eOV2v*aYFd<CFkNjXLWh2_6Ie`wA~bfSz=6I-j?_ zeiCPpfq?^Zkc9earpBao;bDpBjgznIb0Ao_u!}(>!EvHhf4g{gXij-7>sSL~@*Un& zsQBX3=mihhn{hAPyvowB&;Eqt1hOt)zqx8vempACJ#6^_eM)3m%bvq`-CL9Af8|VPK#+!N^`@#GX(h)8q1McDIRXwyA7vV1ec! zEk!H>>j~^{>2F3=TVG{=mXLTnHOykt9hmuqkz2~(rw7tm7FI=GDHqL8w9(=3o4dXS zwz@7^lbSic1=|71ogM9pWT8t7x1%9`S~WbCiS7Nn3hzV=-zV3io?94T1&dL9{kOyT z`h=^GFv-~=t>*wv%Cb;r%hoEv6oKkPIn_*4XRoLj^MD5`4YGh_`M&S-PSXbYxOu&% zT;~mxVE1j88m0AO_>zUR>TI8-U$P20cJ(R1)pHK_Y>2St=e*1DH`}o@lsJf z%)B^jLs*1>@V58y5cw5((Wl%F;7nnrq9f@bvozsD)>QcdUdwv@*bbGEpL4PMwdU78 z6U2aC6G32MuKjgg7zT1XkOjQsZj!jsSQTR_wiSkv13k(vSUp=!SV*RGNb zOL5|NLta)*%_Oy7GNGbFXD|`4+p*U_yu7Vn7en64^Y~5tdqDfa6Mwn`Ie(^)H@M-q zIDXEm`!PMdhT_s21N-m*o+&=Bn^H7xyCig>JTvLu%kUV+&@4cQ;k!MN1LIR3<4e>t zJ-rE;*Lo@?0CXUVhyNxL;6Lwek1dfEVm_G&L}#$5rfX}PjmCP$;dM&_jwFRkkC+1W z%VPNgZ0-s`rO3TrJK>&nF@4{P9md{Obm}FfiWHF$4Pz@10&;6)U!`>TT{2_Y+}tl#K8y zBF=H!QERk^cjK@Inf~MDjw8rsk}^Ks_=UP$L9<))%}v9x(i&qZCU6MF>r#XBaJX*G`)%WjjAZsc#VBDui&#EPF~Dg z1CUBxeMP^x5}dvIHaIhV?)!SxF3`^+|`=BW;?>BE=Xk8Qk+Jx6m4YkaoaZ??Z3b*}i42t2C z`U=U&r$BwFEIbFSyi%}8&$ogY(qJA~b&=DREV8=ZTwf>ItZUjlv=0F?{6lVCnaU3= zm#6|wDHldkHN1%Jicixq`x!GwysX6tC9b4B>7J@NOQEgB)|r4Xb)VIe6mar??rmSUlqt1B~|lxf<0DctQq z&_Y4q5sFd=N~G+_zw02sof9fcwTJ^#OkvUxM+Q4z^E$#d=8R!p-~18Pc8oRkN}8f; z{NA$})=FP*BXsLZjh^6G5HbNmLhC>&S0M*?O6ea#8~xV{3n$@9>$e6E|B~Tf&v=jz zEyr$1)i6so=c<8A(L=i!fMCu>ESi;UCx8vk zrRUb36R^BXPH@**LItWYh$h;8CokLM=OoxyDsi!?Gb=sqjb4>x7u_^)jsJ%r%lkIk z?0PKBH!`@IL)1rVafLAA%mv6qtbFNuY6}8c3{aSDCty|2#v(@>%Q!QY#py}Q4 zj$r5Hgs1L?IapC3T|Eqi|P5G_k|1-~^dm%*^aM?4;@kSTcmEk0n9#bovhKruB+_AT3 ztKaPpNfnv7B0W^?Z}6b&UBHXxh?d`Q{mrwIp^j$j^vRc3ef$3R!~VYl;XlsewE-iZ z;r93i{|9^T9o5v<^^Hb7Hn2;PssfP?g46(V!~>W_0t7-yC;}oTp-2f;z;mmxt5 zu^cp`&(fZu0%1GH*wlbICQyxM{wGux9B$dLJ@NtOc^a-rc|zZ@g3GZ#xizyT7eMpv zp<<02O6Sf=U+a~|rWBM>Y@UqxKtjJ=tIG=IFvG>Q=lw*(4gobUyTUj?AWz{5mn_l&p ze?xXnY&<{pVW7l`frL-jA|e&}><`(G+$z!+X^J$H0p#BLq15APS}*(G*D^>|QcOgF zvt@WtP9y{UFtfPOHEPDmM*q|ssauf`08Gn|t9&yU>a2^#>8{dDa|!U5ea(g)DZWG{ zUR!8>1zx0redB_{$(vT%YekC%`iE9+`3z2>IzC3p?>GBF5%aepBrX{53q^)6>x4Xv ze)ubUI?Jg3#dr0aO>v(d7JlDUx;BcZ&Trrf;i4k(>2Ok+!L ze^rTun?H&;BzXUw?Ec!5wufVT#s{I`d}f|wQIKCvg{I}~W?Tt(fme*Bln%}8Ef#L* z(>g0Eop)#)F`gZZ)EBg>X-d{#>)aji!AnN$D1rv9uDWzKMA%$s9Olkg*2&LaZY2}@ zW<6ay)*5s>TqFh>jpLLaWf9ph$CVP_zK?21oUw#&&EC&%du7V$e{e=olEIhKU}&Kw z;k}roR3PU0^GZaF)rBqv?4D4{WARR=vblql6fFJNfYV)O@kAKIuc| ztgTlPhX0D>Bvy6jBh%}YIktE*jz2L1#@Nl@I-G%6lQmfV%9qmWkl{2K-rI09MBur4 zK!kS$qV1}A5%#H7kA()LDNvE!@)Ln*RSAIHYi-aBMasMsO2piH?2QIl3#q|!r9HlWamJSHYG&|zUuI?G`)7c>*18IOOp~AcU;n( zmX7Ki)iBvmQi?NVxRr$_4TvKs&^uj6q3mpWB#~a$`}1XYHGOpMbZ44?ive>U-D7gU zyDRVFhg9|PY=|7T*sI~a!P*5M>EI^avqSweJqq3~dTaieDLOthHVp_wCZ;$b+=f0n zk(5`xu7;AWqA=$rRy)_!5CP~+b1a=MiI;ILubwpTNM@;FUS;}@(x_9j8+1s__mz60 z{#TRouC-moy9}#Tzif-Sbr0W2QkHYE5uOKuut8iaGjav*@^6|oUFa?y50!CROYpT% ziMOY;jVTf_3S@V2I7J4TWDIO}xlI)5uZgNL^VdFk>oCs>$=}B&ZI)ANNdnbHp`q6> z5W;|$me;AU@l=jZbRLIkh)XVQ&rn6EBd8AP*#obf?g&f3TNYgrXl^)}4LtGgywYd= zv#_zAK*5;W0o{PQp14=|uAn+dmlN4JThX#|`6B_n{@%){Uf%C;W6XNj`U#*{2`82T zT3=*?Wh50Fg)N_x+JL^{vJJisxbU}0<-a)`y&JGU^-JEn=OMjQL7L0fk>bG_|4os< zPVXyXZ74WRnX6`PUhtCBFN?@Bz89+CUXIbv5|EM+8g}z;7MqgJO-z_`p6S{hGULg; zk-3I;UO$2IziExK-#m~p^e92~lexN=EGbL%>WItc%M83Cbdg$iTTyZB_0Z%oqM?9} zY;D=xX{!rPZchx!>jd{f?goQU|JQvna?(qju=h>+Vt)fH*~E0DyW% zG;6P8p0R4=-PmFBWTpr+_i-uZo}E~{ueV$M)xInfRs(a@29dN z;lYB=@`Ty<$T(UNzJJp3>Oq5NtzE62<-#gPO`#-r2uBp^4pjnOygNW&R=#u4->Nli zKtQj@^FHe?o_>a#@6^-QS13;Sxmrf-<9UR)Lz*l zyO|^nD7c$W(@FMWrjRA%lSHBnCOhzo5lh5q)7m6qKZ>h%Re1(>eJ!-f#4SHgrZ3qZ zsK?O#W;w;bs-_|y`839rFBmV>TIBml+1Ot6&r z{RO*oGqznLeCx=*tq;$ZX!I9-l6=&6eTGpC5)Z_9RgZkGRV@K-M^Xu7NANxi9g$OHiKcj8okJEWy*urx@k!f?BdD%lVds%tP zkW&LCP4}6^cBuSpc<7Pbhp)MOrLA}ksO@g$yR z)z+m@9-6SxhlkvwD&=09gmVf5i_w#~nN^PN6U{xbR?NM3>yvSG%>s?^15Z^%&8gIT zj!ud+2&pcnZnDK6X_z?)aVM{u%JO+CuX+Oe$DBPrMK^J^=TqG&gH{*?dW5CLU{|1V zF4oNf>dcG^IaP4)xKNp*c^W9_zCmG08NqF0{Gb&h3}ranlo4l9NvpWMSosfa zx6M`@b+WN}tMsaZrZVFd<09TzcZW*hrPO@|w)$5@hI*~wv@#s5nvKeex-Mp$b~)u; zgq`(pFn3B;K57p+p>cU@!~ZhwpZdPLs3Px5O&`F3x~9^6$ktueQ(b{V*$@vh)HV!m z@W`kM+B{^3u@+s+rC`n!cgZ-($|%0TdSL65fbVF$2AM?q8o~AO@cUW+e>C=S$QRH4 z5Eakj=&UZ<5oWK-fI(~%AzKS#NI5*hOphnGdCSc8Kb2n36&ho2@!abm%$PHZ#M^=@ z9PlfKnNe<0V5J>(7FyvScFf;$GP)f4fPb`7X=tlbjdp0#qR%Qz`s@3WH+kopR5BY_M3jb~*5{Q2l z=)C9s@SDN^b{&Q9GFIbnofXP(e3SFLef*nt86UIN?+z z^*7e4S)R`clV%R_Z-9`UXE2u=0I|Q`WB>O@eH?MNB%YFZ3Uh40>Vz+eN3}4oG(9uy zs9Da&0X;OBLiC3mZ@4tj5#Ea)c_Qk8K=?C%PC3xLKMEWvasxl-jNK z^=p4&=Z`9(rk)kzTIW(rKA-;fU3oik!tgO!%dlRrNbPiI2C?@{`hR>Emfc*F!OF`T z7d(?5RR|Rp8HjAW0xQ?z4~urw6)`NjXkV(ME8?cjr}dM$^%qXfx;-6Jk?_j%1IFI~ z++W$0f*k=$tMrO%R>lg``J=569xKa$>z3(vi{n5zcIrD&m*g8`aN}YavI#8f^(?MV zbL~sOy6MFG6v1zRm*~_jr1kdcyxLvex#O~P_r6p!eIG-&lCH|Gi`OzH^G?j3Wx9-; zm@HFuLXFdtN)S|D!>RrBF?I%R=(No9=68boDq8JF7_2T1xHZ9-XsI(NX$dE|dOOl5h)-Xp1(Q zs#XlfJqaEib~_Qg5wIeobMlYkhJUoLFa+16u10Ly8;gMqSCX`nCA^m0||ON^IE96<&niA5?SvJD{+Habox>0aO5WZ5^s-TVco-D{*>*m>@)2m^G0 zQE=T}W(^*L7@O3|kPs{38Ws86;{gxyRX*ibLWSiRxjLVLgTH!k3m;(eS~EwN1G3f+ z#jUH{x1K}MTu;QL&y;|Q)BGUGLDkEcx=)5a^>X~hTN_Fqb6PpVAh)nlV2oHrri@Pc zzn0cEdYW^wmiIx#(I#5gqQp9-DFt{WTP7hP`h)KFyS0Y+B9ax}oXPTzPJ>*|vUC!< z@<40i)*mAz_`8a4HpLNy`N9x^lUg!}i$_JpavZWhUtT-z{K2AKp%UH!4?3)!*`uvYBX#B2;__ne%MMMo<+$|9yr?8XB1?RyS4faBh=X~*SEY7UV>>T4o@-L+c zz-Se@Y1-2?l_ohCjly`D*um#7YWjL8FY+OC?NrwZW!X`R1{^| z;+eM+L^eavi7sT!R!j@_+}stSopUxMsG=O^6J?2Hy`YAA0yT&Me^~4v<0a>!Qh0+t zGF^A3Y?xSncTg)wXcaCGuG5jtomGmmT=_h&y!YYcyvYVHX+p7Ahs(2gElpy@ zjVzIDI}ufTYqcxF?%(QF*i3S9vPnl-56^)o*WQJ1azEC)s2 ziuR6v_Xxn=0(kg$q+47Y@wcNraEPx`peoO$iSQf5yVoP5oNr8o{C_~;2sku^yKI<& zf@35p!N?W4_a*MA?!RTrEDiH3=D0{LimxN3A z4(cxlJQuhOIIhX^(sr_9>$|L@<-Ps(9%cs2q5~Fml)sBvs0b!Ytnyv_4KPaySY(Yz zkL1eW?RDAc#C}Bx$sV~>FCU~ZkmN5PzBK4(Ke76WAb6^<=o`R#mT~E!4(R?GO0vnb zxH~j%_7v3|a_^DZaK6;y{Kw5#LWnWmAo2;r@x)}PUWe0N^Wo1GM6w0E^Q$Y)rqu|% zfC4u4F-8|_HhHb1k-UnS)Jv(e8WLkYPo-ydGU@I!(WHbEeWN_5m3zrfyTCl@-&l-j7;aA}clDD7v2C$ZtiRy^F^oVH`q;gw|DTal43Rr&HM_X;=9bc z4Ge3hKt~(MriAOT=3lC-#>7ACq-L4jtAp`Vow79K>pcXZ!NBIpj;Qdh{NO9nQe$A{ zSig5%SJ6oOsG2I(fjGjCTku7oK{ z`s+`uBHO)r@wGA}VtFXUv$!zrFt|j3FvKMEg+)a?pr+0KZ1#0&s7IGE&hDEE?9+Av zFCesw%#-TAhsv^n1>_yLQ+gEtR1uZOK=I%?MjNX+Qhjr5&WyI$Z4rtH` z>3;e-3y2{L;{2ddrXWB`KmXGRQvLeGQZTz#k9J9Mnph*sS|$D~jZ85UDl0nkn8bZmzavd@MTTkJJtw~)3ZJTMTt)#!8D(dtTuw@Doqyt6a-@Hj zU$F+H6lzCSgSJ5Ryv|JYHE+fU>!?kdp#7E0{6*tFt^uUHbpfC8Ps%9wx@1J!(|lZX zA1zSCEZYW;7`rFL>AcE}{xoR6(HQ(DmJPoNm;WQMuM6H?!k=yYt>I&EYsaIY#V*{(YnL0fRLS zc~7N9ckxwGSXiQKM9aL?di{O<3;^9{gMJ-KS@B;9I=;VSiqkdzzQ`cE9E$+MoP@=Wz&hSP1AB%MUXN}Dr;O2oF|=3AGz9y{IMgQy(3 z?~*akRlH%?%eLk3+xs$^n-g9l$$23oi5Qfm+%=l=`CtThM`_h+;2!Lg3l%k&5Cck} zp&p!H+rl}Rn5;nUAuVLXYgg^w$4+xpAXzc1qVup_^4-n?CKlth$yMUTi=RYc2&z42 z`s2>EN}dMm$g!46(6Ud5(x-neDjoBa_ROIA+LMm5{#J-(9B{xc)J%E2WPdEH~&2{q_HSG27KT$ko?aI$QWK6!XUg=dBVI@8M^lpaIlq`xC8j@e&A_g0!>mAj{;5|0?u;08z5bzmE2e(!tjt0g ze(iFlap6ZILbB<_GO(nAQs!`}p$q$t!E3Akil@)tK_w><__3y_WHRGUZY>KE^zZ-v zU|Fq?=(r;8d3k&-tjRLu+0Q>2WJ18ByxYw)hWb*+A4f?fI7p(b10C@7mMQscwEGi> zf7W%~Vfzsc*Q7`oCp62<>nKRre*;MF{5lT+*kP2iS2{dUZ~bj|;#>NAx;&n$-8IBR z#4AHw1C$3c%89gt&va$yTTJJSADj`a=z2iX9Tm>nKSKULvj3xl|6O(fbhSowjnK$C z3NyteS=u|F+@lx5qhWLt&*$kW$a&GJ_s=i=4A6FGasRv~?@6wR2>&r{{PrO2j~DCx zS>|2i-Tez;vA46hZ{IH3=cxnz`5yD`NxKoy1&0ySHw9CweXDZ4F^?Z`{8|bhV>&Agnn)0WwEgrA0J0u17GGl^)Y1f z*vq50D9#hg7i12WKZoCB|MlM{Ar{MX*ki%!TXMwjvrm}HuCRQZZMwle&Y%F(IK9FT z-oXfhgGl@!W_AS_A>33Nbk(E4Oou^R1b>jY+0sb|5$3d2uOJRZ%~@XmLOm!*OB-es znOHyMD$4?SOhxaRU`w5#gs=x<&X-d+ZVPs^a7yRqHo898mZepz^uIU%@SF#b0pw}$ z5Xq?MZ2XR@%$Pi<-Q;E%veaMm4_Zk+)))L40K2G3k{F|qa-;+5o)~4=kOFVyavpA0qy0j5|sYf@3TKd<_`?rdJHs){(v*(zIj{7BX={NQdF%q{p zP-MU_WBYpvHke_cMJCgZGq%6xM##|I(3EO%9;Bcq7Tp3Rww~8t*aO_$XG;UOTr+;0 z*eYCRKVJQs{q?_T7XbJV_RB)X=p2kFvlt;RXyekpk-KMl&kC4d@z}qW&6i}Y zWT^|MmU(eR{6dCbv1I#@pVppBGL%ZJU)?VmIv&yylzdb3z|>WFrUbQ&Bs#E#iU{as z?lm;ux7*twZ-PeN7r9o@L_2OWOc6bv%|%-2IYAH{X5bt5xQ`-`w3Qctk*w6ujTXk4RLDEY1BgomlG zzl7Uod8r(6n8rP6eqwS%qBPD~y|>@EV}{gwuA>vJ`IEH8BSzj)oa3iG{k&|oY;@zM zsz1DmW1DYjpIK$_=9%krFs`g=^s(v#M9Xly!0O)4y#`pha9D8`;8z2R&LNHW8<|*9 zxLzV()|6Ctqs>DgTLBzgXvdp+2Zj&WvS6iij+kjv&C>+n)rOnO+r>4 zzeYcocV^gJ_xTn84QjV#EZPL z8R)CY_PKHL94!+O_Bs9i?v)!!Q;DG(XL9incq{XE-z;I)Td41Hk&|W&*yFRDcj!%A zmT}FV3C?)*oRm;=mX5K6!+Gt!UrbE^0JowwnuhD(O*5pFnO64hdtd!T+2sD}in&lv z-S}!puW{!ETth7MjLPx-m3zrfBm@y>QYl#;+T`3`qsBGlF-|9ys|JVqzfWQ7T9;hZ z8t%vwc7=>}byHvnv@*;mqCM=4YlI04{_Wz^SC1E4=!db}Ije+xrA6br`M6*~XFZ6AE8n^qqM6aPh2`m1JM!C> zx`t_zlG}J4I+Ampk7~{e4C?2-H|>OF_cWFEUSIGuDk`C%l0h6K)1xl;V3vZW)=O4d z9T0J&M~2a?(AkU>x1MV1m(WKk=B~8Cq1nFVN)#hbX??B1BgVYLEp(V8MLh3CLR`C3 zRj{E3)WG7#nj+GocaW9!$1^uJ(XGeDNyVXx7j)OG?;2REJJ7hWoTH=FZ+hgZ(^``; z9fj9IU=gL%gJ5^wIx*4Ye6S3D-KTjo^ik4nMZYYX+qPb}#IQk88eBol#3N&_E21O8 zFvL%|*Jt#Lot$A8-3#U+K9@*@(=+T)@j5JYQ&YM9ns5o$5{`Y7yv`?y6g>)1Zla3Q;Gc{@@W0%3st9yIP_l%5cy*Sq@PlS-XRFq9(*y3V1EGRW_2;#h6 z+2%OA241^vXH-p1%$0+ThEI_|adQgHk#RkULdm$DQ!#5JTA;P{9{vV|Q;2j_N~O_W zL2F^rvY^D1X~|$Q6gr|F2s}HZQh9aTxNE)_)qtRCo#>l;5UG=_-ZR-9CeC~t3RYO{ ztPa+(t3;Vpb}Z%1&LN~pPU-!_E3=)C!{$nv(XF;ag70l^tsUtgb93|FgUR)BMa`ZE zF3p4@Dmf2x>Li4vzw8Kk8?sPcV%j-`5`Qy@W#p{)4U|7^!}BF5q`82Kh(~K6$ET!S zqGP9@Bv^7sUnye6daP&8&v%wEXjWsxUCTjABg9H=Z9CRpC)B;^f!W=8l9Lu94QY%9 zwV8ztaLwe2?EK_3qcesW*Gb3&1{-QSW1BVV2&+h5QdcnhjC&gOLnPk`++Efed@>JE zunBu6%$HUMU+hX&K39bc$6H!;7qO_W%Z#6Vw1H|^O)Rs5UO?P-XdtH~^0*ER4Z>!m zdhPB`&?!cFdYv>Fr+9P4ooM~NgM_hJEVqj>*{EA@K;I}Psi@^u?Oh6sjb<>JqVMS% zJfUCv+;xm`Hf9oOkP_!cgnAe|03`yHvzuGp+oz}U9{AFou33QQ@QRkSiWaJ;df@;K zsgPaDC`wc2eo>~a9$;gyKIFyS7jUd5EYIFur}(ktiyFFwvjX`95+dnhHs()Csh`C3 z7?(CgTJ0H?P~6{5hM`M4WNXOxC5Po+R7l&NOH6R}i*5qTgdSg7a_7J$ z!-t`-h*%t-84DN}BUX^*Q4s+B4tl8e8U>b4=?{(M;4+=l&;Z>)R*W!@x zNR!T-RYkQ&6L-|#Xr%Tx@BIXZ@fOp4*cTz$U43Q6Omu&Hr>kD}^Vs(}cJ5p7xB8jG zODG2)xZ*CGZ(;{c!V<%oTD5H+mGYAUR&pbE+|U8A^U#2HYv_mVZI=?M8fJ(=qAV6%k)MO;MO#wNRX^Sz2m(`PtLmy1Lb?k=j!WQE{K5_1r@ zQ_&{}2$m@tmgU)?Oam2>%(A}PyImB+H|4%=WbR6P8~ZWwirhhYuk0Wo!r1s_w%7vP z(A;q4LtJUbVwN7$Jq-d&?^ha0nVi^ZD@d^7sie8uXVrVbR+4G8GwlqFDwXqGMJ44@ zr?VD0>$fW zbR>_;E}-D79v;+T^)Nj@#O;x**q!RH=dKOS(4`qTZVI?|%4&HU20ErJ=B| z*i(Z)ZRD|FmuM#w*JP;gl_a}jdBRz=kF2ZbqlBxXNwb)~gAjvZxZ-@%<&A_AYM^`3 z`mR<`n2glAEYiZX6Elb)792+zhlF2FkA}))9SincHZ@Kdgt3rBa&DpTQI@8`*9A?C zRlQD=4r(DfjVRiVi7pM2xxSksEF(wOwSX8g0%}iZhdRl2upAjlkRO^Eb5k?<+^D@B zk3Nt2jE6dq0-1-F$6z)&$xcY)iMf?&;8ThXB(){=!TiZLIYdcl+g-EnJS53?GX`s9 zOp**b@m#Nxq9bRCV%d@sL98PUY?KD$n9XeughTgE-?s1EnH0DOD~>zL3y1E6m*A1nfx|bl#;2t0 zJw819s8!#=w}EU6-&v|tC~>H|zZ&!zn9OT5T$Kc@QnPk?hTO649_hf;>0E;*62Ae(TiiB*JEjsSy*BpBa`a)q9GYz{p; zv$a0swI*hmCQu}9L^>~~t^MHOD|$)bTq}pT4kF+w^Njo6o^8`TCEh8Le)pB+iFt!g zOutDs8H5m-)dL&D3?LfZA7=Xn>bmJv1U@ji3yR^Ms~l9NbYzSl6wAIhXsEbmHI*Ni zD$=?YL!$G#ks(8PkQ>VW+#8V|D||rnml&FpR;5@6O)CcC`qm)nEL4EOnlhrsc;i69 zjX}Pfhz$!^b3EK+(TUQT_xNkxfE|WYrf3&FR$sqlY_)jxQ7*-3xY#|Ll$sda?qlQQ zr54{AILmMkv_JG}tlP-?V@6g7VSofXp+}D6Oi8ZV-{g9QnMdhf$F)-7_#<;h^q3u@ zp1v)SH!{>&&B)oTRb9Q8L}yC&&2DloIunr@Hq`E*4>HG_Sd4ZF-m@uREO}HK87r^80ru3+`Q8a~vWHC?Cqo+tN>}Y&&uKJCPSNs1LZAZ` z+HY>CpiD@9q`vGpH;TJ50*JDuXhge4e~oaAnkmsGej0N*GO9yRawHvDK-oMc(S2lr z?PqxY4D zW2F1o0>R~y>v-6MgrrOhTK!Y>*TB<X_R(~aZ2@@53ZH4@p>kNkM22HV{u%0#t1s_WcyTSv1=vlt z%9DY9M&xJfJ!j3-5`8iX3EnOo_naV`XE7(KDdf zw{2V3Zt!R2doAi4LphWWmsUDWDKK&SWaTa1l2>~NO%L=~zuVhCiq2SHSRTlqcVE5p z4X}IGbwTsjZ-8G+H_pGdI02hBO4b2mvs+U`L0+%dS=C9D3NrR|KXV^QrO@@;Bd{ z+Bk96tdP7L9`C8h3oH>Dk`z2N$Zt>onKe4*DwF48q&x3my``P@;dLH8iz2pV-z`1n zOAwZs!S`K%_a1O-i(fr7Cd2gM3&2}eNaacGh#$d~Dn8tjX5^2}09{xDi=6Fu8~E$L zAA6P|8E(J6{o%pAXuK@sEtLNAK2A>aMV;f&oU2oowW!cHsBXoX<0Bk@6RU z5t=>%P@nDQ>$m8fDU+CCR(b|&7H24V4HLf%jH&2iOWq6J2|B3{_&4+W>(>91qyJ>I zfPFflo^iiQ^(p=LKM0|bpA^yJ4aa$#@j`f${zu9ePWa4l?bV3Su?N_SUVj*f;@`}Y z-hCQjfZ9F{;=dRuo0(A0RShZ+7)QJ;k#Zrmaa1NOonezOKhqOd0++IUxNo5!IR`dq6p~|o0B#f zQ3sfBCY5ln7(JW?qK8l7>{5 zdpXZ#(kGoP0BIj+ndi-MBHVeEd=AJ1$tX;r06T{twXL7>sfu5DOj$mZ_O^RRjo8=9 zn;CN)NrN_%^-ynGbPT_~a(0^!>c2W+WB3))J>Nf*P$zS~=)PjkoQKCYWNCft7aMq< zv`Ml?A{;9Xywr1Zv&mrHzz?<$IsADR%1G zO(#99c$e@SEZrdYdV;fbVS%{H>qa!PrTHq#q$2q0{nc=aSY2-K@OioGZ4al^Wmleg z)iPq%^U}~vCkpiiEKE_#x^JZ)-1Tytn^Vfe)vcbzD64Q4?yQh|eqTjAfrRS}A`g-ray89F?s6J-JG6%CIM8Z-G3UR9@7>m@zc*P*G zCg|HYUtf6fo7S!}ZPI5!GI2gMMYvlvkW=Y$^XZna?*F*I<9cviv1Tn=Wp%Whdpxn z*HQrBrDHgJKPw@68DZ$BufnAPrQcB`QbXphn!mv|_jUgY;0JyK00wB+ji_&T^eHp3 zz9duRM)y#%I2jH-n=B5K?j%$oKUkGE4*sf|eDZOQf}U;1rhYQSgXU3+HqN?AF$&wH zs9w#ra!|#tAd4n5KIry*lFD_=5ljRvmyqBHF-CZRvfKT#Dg9|#FWA$y=O})!!=HgA z#|F@dyH+N-Yr8rweAIv{E*DEY$?*a%#ADrpv{0R}T~e3aBNNw8~|7f9WI? z445W@Gg|Wa?iG0_geOXP73NoV=Ee-taS%>ha)E(gn&LAly{`(NJ3&7~`MDxjTw!%> z6!#n}tAV$czH9Rc@)1UiWqJDO(+tABl~)W8ZQO_G1m5Jy#YF)Koxbj55=+w|U*fcb z2`Rv_W*TT7mS)W%wYX%@U#9ZLG?&Q5^je_+dt|XLJQ^bG{KrcgnW?9G z=X^7({V9?MEzPwmc_E~$j`oMh2wneQh?k;VnpNKjd}^%2Bq?Xk#NOBSbFy*De0!6o zW3omy$~%%HzL0=XluJ#j90Uml9>gE~24FL@x+e|eaL+pqG7-!(g16l0H29ce?IRMI z&KR-O@lsG^myxI}=uS@0uc#nl!BmDJfploLXq>0Py*4hf754c|eaVv|)K!5tM-RNx z6+DTg2;*b1g|sw&DP9?_OuX%cxg|2$lCN`kxhK% zkp@^sM!k>uzT@>PYhMg~^WnvtgvOiS&wt+(dv%+AMIH$#7Y4p$FJK*%DnA-{5m&-S z;CsgdOSb~DsP%O(`sHT3VO@jw(yiLj0>i34X{}awM>UzyzVmayMYjKkl0NEr1&hr1 zHw{l?A4C>)(u0uT+i*sP@uQ*4FPX5PSv}mR1&T5DE1&}d(Tb8}{?UgeLEeo&5|*K$ zWld>q29yM!TlT6V9MOuVrm!z# zviMzl;xbPU*r>mVnRnz`FM0TaO_X5FZgws3;A?9}s|D|K$9cxPLXJtj;{0f-p+z2?`NM6_z0nv7=;%tyBD=-Cdk$!M-zc&~-Sj@UA znjO-UH9XYR>KRvT3Q1K`SfYrPy;X73Q!1x-s?He}{#gf*IoFvr*_`A+A46H)+ljx? zO#&+OKF0`%FymVG_I_by()#^w7Y7Ut2!lw>DQ`lFlYy#Z)-lgu>b1+{sK_UHTdIXbuwpcVSBo*TExbB zK;7QY;FS1_ZlesXIBHdB^r}*)O5|_{8M3NZ^@<>jv!@tNm=LRHlnOfGpV384puc);F4=6> zIR%*tA(!8oR>TkHr)s+T*VXZPiw_{kkmKVX6X}JM%!hxDl%CUv(`XYG!An|CMW7-y z+bB;bRDW7a`qFBzdZUMn?CV~gN}lfIa|p6Tn>m3anqXS#{nv8@Y|Y?A=(VcXmxxVM zt-Pi1Ek|{dh{!~|?o&nJGfl4-oPUm!P;wnFUBZZLk~C-Jon$965Wx7XFrZzByz?0C z$Q}z9c{SR4uCnJ7|GXK}XsV^3qzZO8ubi!49Z$#m!V?}* zPS`!!#nbxlFHao!)3Rb~mij!1aBdEJ7`&H9JTs{xdR1iVI*Z*<(0Q=`N{^uT%6`yu zmnpU4okudY-+hG|3Pi zd{Njf(KxDv6ns3pwmNvGX61}!`_wWMt32jlz|1Cf<=+6mju0B)D(hBFVMbmN^JWl6g)fF6UEjZ_l(BGKvdsE zppj1NZ2cA%^$E>)q!cBb%?9u+*hhNifScQusyw!cvb(;HF5>jH&@Sfr5cRQes<{j; zUSS$-Hwo z+k$JAk?B3zD($g66|J`JJJ)x30&HKeS|Z0RW^w*40ZJ})@yOKW$=!F9It3?!`s{T1 zrOY|7mwj}|fTQC@F2=*hS8Z2?hmBei`-(qDeZJt?u2rAL%&IU)fFkPRyYAM>LyU#m zcn8-t#{KnuN=;tLtlhACZPwwA5$!5`Il8qnRd>5zH3~{G5il*`Iky8F((mqN-J_cOmOzG@Dc1DZkeRy{H zQ0(4dvx=gkbhc=xs5uB@+0H>r-vLY{S6cg+p%8G(R%-jj2~L(5!|~Saa{|1vG}sp8 zLGTkYzV@$fto^;7GGd3SmJ*|9q{UULG96#Wo4vUp6Jh)E_qFoDXgL`!Bl#&iV=3Xt z9>-}@6;#{c#$HD*}cj43tTw^aC?Cq{BLA#5!Z#g+zgH+~Y z))=@MH-#m-WCD^DcXD`mjw`%|NJh>j2F{u+p3@Sw3S{F<*oSQ$JNc(;{a zp9oWQ(AA^z7RNIDQC4;}wGw(QQ<+9@p&GmOmr}u?gou|NFSD3!xffN;KQhw}Dwdpf z)+=kVU;o5hZ?g>XSs#igqw_}_tv>29vk8-~oZ78dpN|R@QaKXiw1KfHe6mmCGcmkz z$7`O%BhZFxjrI5z<&uia8_g=AKCb$L0;h-f<-`0Q&XhaJEcI#`&$LySk#&yS=p<03 zDSr7M$5b9P<=~_hExUrG@8HKNZlh%5X=qN#Ct5W&FfLrK%SQ0kp*r>H;Ja>Lf;%Mo z2Cjo?AJ%W)N$EQipfA_ZGTha0mB;E*_FP|##z`#-4swNdT-pdvpA4Hf%v$Z1bDd{P z`38^@EM`?QHr5xUp=&@z-CV<#x15j(@kq+Z7Be0e__76d(wCJzLd$*T6kuwcl4i%- z!Qz#uk5fwVY>qQhbMJ$)XU|3G9I#En!eV$3MR~P-xUOALpI(JDvLwcdu2gmVUUa|G zfYo&P;1*vlcNl2}<#H}_V&PJhCCWRpyvsw<5;AH^HCe%1CQ%?mV;&`MRr@|~Z9Z7N z+SHAYA@QM*m>4!WgFx=vaW?Ws?^!Xu5MJqZwzX_#cW%);HTD;mqB7D1n8FKS+-9@t zrM4oJCA@O7l7=)$L+Qt`y(;d+psqnB?Ggzo810;m4-d%X)0T#&N?muGaN^mKu5Kh{ z9;q*x4Du&Fj@5FY^-~4B1H9`Z4(37QN+>%uXb4fkk=A&zq&FN9Iu@&uI=vb~Td%Pr6#W77!{ZhY=i*)f;algx@8%ZI6L;spC z==W8o0;oIEXSS-a4GT}pD$T~(pk(mg-Y(Ew)5q5R`Bea{g1{vZqEZ*sSyquJW8dGX#4q66j{!oi;| z-b%*vf|4O;iO5Y>!a%)7t;9zu?_{SQNRN6rlVF^8jxtwFg zIxYd35?cfl8{J0sfMUoQvrEb6N<`Tcp5RS-xs>1L%*f}X|u zGy>spS}?zIzsBicyBhebuYl1IhF~I}l~4s!$^lskITNF+GN2Jf85@1dt>>~}h%xWxg%gDLLU{pVDXzvVo&USC9tHCyd#ZmpM zvI>$*GM%YH!w{;C(-D8()-a$d*vQ8`fRz>RWaf9)I!&nrJwDAynTwqYsB;}e*av%d z6vi9uqJKI>z1*bZ%cBUk<*Mi`;q$YKNwYgJzYwz zi@~L6R=%8Jb`7ebKyEDTMt^&FPlQJ{BVzCLOu}CDb&C!eh576f<1AA!%!EFvlX+I3 z%@v(9eo~+lTN|y@v9M-&sZ+0GMrWwLJAtIbO8p^O@*r=!DhDYqquI|&P{{rDjBbH& zDa-0jS?N8%g6~2AfK{lpV!gtUW{ki=nnvUe#!$j<%w=bRzaGnvE2uwoe7!_Ts>8UQ znXx5s&6)c-zkhvjO#8W_HSY3kwYVcyGwEcGZFc?)1&+9O5u}Xz2{MMYG4h-ruXhx@ zAug~_b`rEpedlDQRUZxO;=^$GveNy>*q`7J@YQoefBldPUhJn(sKOIlKUz`M)x&I9~XBx zG!H8<7Gm)6%L-Kjh?d6>pOR{i+W?)IDnS%abz=7)em<9vDXK0D=EV!YViuXhh#09J z4Ymi39lh=aiH?nFu3~(thWfpN#MA;QY}dHBWXWUuIY`QNW{moyqRvvMIIT&d3oA=-X04a+B;0sKFoQC|Ny@WqNqlfV6i3``6i)k~sh!xucOyo;t_U3Y5iQAK z>vT+w*MfoOdGZ7MFZLg%x8G+uz+Wqb?5e4o?=e(hAZBTcW#x2d#U9YhR=jEJ&OplY;Xa2B-hiHTzD??Y+M$j0?DP z{>-hP)TrB+zu>!eU)%@(dTQytU(sjTyGs|4-HWupEIVx&c@ws0xKOX|Es}Xx_(b4E2T=O zbJd$Ik*NDjL8N@yX35JIGc-b6qNJ%l1vIs^gf3aH=c^E~f<&%5{U ze5dU5JLiwR{+QgeW@gQrH6?4^>$AUVq8PA(~YPi-jFZS2s688MWOElav!LPT3sxKV<*Rcgaep__0`S2$O zj=N8t@d<;vJU3fCg+D#0VjDKH+Zc-H4g6EQ>7k2QRm7@DXwnY=RaPMA3)KPwu;FOenR#rNVi(NQ$QqX4!*j(yl#%RgCtA0hY_p)~#(9pg_g zt*<2pSWo0!ZLhA1BSL83OFD z^F|@MYJ7^SN5E898HxzUR)3n;_s`S`THfp+Y$f@D7Ij2|5L3^#9LicCnf3jk!^1F^ zw#}@kLkM+Ng<{QWOi;lhM1_YRjff)1D+xNb!gCG^>+Pv;sZm1;Pp{GVa?V4a8wa?$ z9=z|X%1gJZk!+#cgEPi5@Xb(zk7BJV@7wol8ef7rwp?c_033+H$_oH-a;6kkSEQKEvKZC2!HT#jPviwBRVXM@s&tgKmnDIgD`P-Gb|Thk?AZpA1*^ zPSJObx4(^APg3TM?cg3pSg?=hg}f3n=xrXgr7W{PkT+O~4N+C&gMa&8VU(=|7zv#P z@T)Ka3@ait1;d#GVBn7ZMpk=8UOyXxo#6{%$_^Vq<=U9rY~gp`NpU1Aq_*w=thEaZ zTdoa^F*ks}&k2-%F6Wj0KDf3tz1w?gc$q$`PmonAM?ev_V0{R7)~X%*T_G6ouW1HL z-dJ1XFzj|Ei$)Q3BZ}#C3HlD!A}z6h{b)hAj1!hEVXS1!+M>scJLJ-!s)3&mX-2UJ zg7@mjYr(JvuJ4}TFFU`Dz}xTaW+rl5_zxm6>IsDt3$l#Wb#{tBx`2vxkEo1SY?;b` zef<;=`kw4`WmNnjNhWDO-iCWMR1@GFo9RV6cwJuj;+vER&C&NQeRS!X^6F_`4ZQ*9 z8ycmR#0!E@rzwE{ZDOM9YJ;0{X2caN=Y=c%W{9t^+6E`s<^ZoW;7233-nufBtnyq3 zdKr+-uFG4qb45bORZa4)<^XbCMo%%EFDRa#-GO!62tR3L?MLeawE9Lm-cu>QGR->@ zClZ&=vg8BVui-Veun6o;66wV%fGslYkM@VSPw0qvpi~o{TPi&otnpdtHP^dADyAEj zmcwfBTJzJZSkWrY{=60Y^A!bch7hr>piDo1JCI0ys7ZN0kuXR+l{#sFcXnA?NwT68 z%s}Qv`IRJ#l^&k?jZQ2qDD1_`8gkRJGe7(Gq(*t?^;XrFl3++hUw5Ra|CFKOJFr=s zS0V!WzRHsA?U0p)GlcJ40l{c%G|(HbpjnN47hEGhAES51nhu@OAR-F}s>5@7V!+bR z*=))3oIIK;Tkkpo+CO%)X3Dooy`xaN`BPmSsj*$b$>Ibd*5t!avgy2kM`f2a)vN zz7+%o^W!1T_(AV9sd!DqDL%T&m#b*tHufh&w`X{}F))y8exDB6QssYOtSx(f&Dl9w zi9DC}&AvDWr0(I9Jtd#o+Irt+veU)zY5Bl*P`zvaR;xy80DPbq1vY!mF~@n{E@Ueq z+2Y+s+eu;PG~#Mj5KJEU)GZBWM7SwH%d`CR5Pa=E+T-Lb}eJkntl{~K-88}gdOiVDKTty)=d3VKIZwhzeyztys?u6; zLbWgYUP9ES?%M=~TV0Sg34{XPu9!dK)5{P_z?`e~bhDCkI0=RY7(;Xl4KX6IutZ4r zS)FdlJd?JxcZ(l;J&`LGK`R^+N>Tj7 zsA6a%T^ty|YO|EQyDNt(zQ3?YO~M>YjIPvdU4)?KBNrc5zM7|fz z1)VZXi{@Lx@pV;(4PCI3a`EX{f$b6DLtM}DiCZ=D66l&|I9OhFqq4YsTw3P24P~a} zM^d z+q7@v$HeqVa>X{Nn_U=|Ar#Y>bWV@AwDFTU^zcuH7sbh0wgGxZL~h3s3Q?2qwkX9Q zJ$CoxqmS<~X3xyOtVUPUWxv~byDv46$RvDs#%j(9bp!B328O5IR&JEuAAEh*HxUS) zpv>!IB}~8U-U2Y!ABYxC?1*l5M_9(yU8vMkC%rBb-v>*!>8<5)J=5wpkBK02n7o%ck_~&HamC87rKXj?8BpCNWRF9WKCjRRux?B4lOS0TIAhTMf-jg^ zb_i$JCjic{9W=x?|XD&99xD0DF-U!LzzF@Nbd)n0wA~7X(Dh@E;cvk zQCd|-0}=!aH?m7frBULZeIEUqc8b~1WT2?PC>C!rVnChqn$tqdcj7NlKy|G}DoEW6 zyS&@2_1rInEnzs5lXF_0F!N2ifO>RDjEHmoM7&j9W65Q3K;v1b?X@jcxM?6Rd!-)% zMVIP>XVUeg+`U}qD%r_J_m&l+zK+TNWY|3s`hzQf9*ZVhWpo5ef8;Fz(89&T)_0Qi zpI59#TMxaS^9^yVFzj5U2d4D``XvV|sN@)}9T_|rx>zy&>=f|HfpJm=lUqcsI1cyQ50=-i6z>`w*Mk(m-=6pNb8C}C+0kSYVaK{I&L z^&~_AIWWNe=ZXmMvwxS&S}{OJlDw8%2Qf^Oa(eTv-O7P}h^g(h*a(9{r&XRyh#>HE zoe+|XiQNv$n(qDH60#EOJw7M)HJy@df)Ws>Rpba$EM~C~m5#1`AZh_&NB|qro6q3{ zBQA*LH@+#&L?TGmV_Mx4Go~maC*GtSO+kU$n`M%6Gj=|AFGPRJ%q&BCdtxmtofF7p zg6K#{DmzqSS_a1muqjpH!uPNd5%a#BO0uX{XK*5hof`o*z@2YZx5z13HCyiA&~)>O zQjV&HZzHt86G@BBiIF}#?QiRf{9Vj`GQ@irMw9DlYnY{qZRvvd;CXe=r{J0}zgTp` zBfFmrs{kHaw&=YfI_TqAn_LRA+Jr5iW;t`>V7|`6Yw{-ps}`N!XOg4jQwSaL3U7r+ zMrLqUmk)iiu*Hp5AhEl zI1i6S|0f#lMoqs$Mi03R8h9Kr4Z77+LcrTz-RBj{PIbH13McQ>%}3Z1yMHVE{1K0 z;B)L{4ay(9BEORZpD3n@X=`!At_sJ){yk&d;lV#_4|xE!VY+v()cEW=Q(?$Oo2MON ze@VNpd?=mP{VnWz9Qteg@}H27q*U5e{SF^~>-vvzTjdePrc)l*EE)dUXytBVYia@% zY>yl{63G4P^l9b~Za1%81JrIAnwrLbWi!g>)E}}u&Aj+e`H}87O^+}dw(E;Gz4bUe zcAAsnPlHWT{GKtpxMmY@4Z=v9u8a8* zub(n~`q8|iDM7wr!++e~?^^xYJDKRZgu~w^Y?Jrg>teAOmza?|C$Yh6;xWr!0$Ymg zVd*g85et)vXzcL2io8$Jt;3YUKkE$dh%!5B36FZrz`zz9`B--7<7h9{IH}Z!jCa>M z`Z+Q9aZ3q+B}$J-x%)oHxg@nvm7&+hFg>viVh=XhzR8u0}E3rv6O^5_59&n#~H8&eI`B)Mzq zP>gkw;8wH)0&IQ4+St?m7*}nAeNO74%t1VFYnX&4G-|Q&L#%xLiBjoY0xTZYeX%wSfoF9#=^UsAEen3vjf?pYpOA0p*sd7q;(jk6g`awpT{%DfV2#H zkXc8Ld^h#lzA%I)A4iDrMJ>j6-os1uL68u&YJo?$bKp}W!}@@FP{wDP07x#Ww!M8j zQK$s1@qEd~YFHOqOMo@gW55UPuHZY+j0sa?S>reT}#k+jyIdp3pqrpLR_y5-4|Jew^B#B1q*>F%J%&=dS|VarO5utSSQmj z1ZT*e=1{(2LLxNAx}9|6^ZSyL!K&-Sh)U#=aW%?9lhZWyRx|qc1UF7>`+Z|IX)5M$ zCdt{)(}WmY&SOWZz;+gF9t=|!qmXWw!!hnws&W(P@it!PFpy3~ogK$I&(}|Fz1x9s zmxly1SaoO8jOqps#vA~aRMl~DQ8xs*hxS*<%7KG&8<-$L19j%N25At@_k8A`4hzef^y=uH zqOO9sQ{RFmn>q;1$!hV7GO{A2v;~)95EaG$`iI*}vH99q8>zLF2wjEs6H#;Hl^D}% zSn^VlV1~n~%7ZOwa)WRMCZG{Yt{@;6U*&KQfF7QF3lgtSw=&T#>GAl1Vg_TobkkNR zx(5@9@Z8uXyHXwhiq2)%Q+lQ2N%q}EPJI8UNoJ@Z(3tDRyJ*8Cl_%FJ~lQ& z1A-}VRu%(9rc&(7*8V-PLC>tNd-I5KXOSWowoUeYN1va?PLc&|awwPb<(gotbc_>t zq&wymSZ-T<$)XKXP?!Za54O1exhkAU=*w_aw=P2RNKN*PU)?*<`RuaQz)@CP(Jx^A zgUH>bE~{uXAl?Uio<`pUG}vu!mY`JC$gw#lSaNHCMYngQn*jh=#_&{TY~ALl(}7Pk zB$~)=jrLckpL1VWbKU-$L{&&=NQF|^9Q#vphtNK=liicN#78RHB~>Ym{ixNH6DGA- z^^LqGSx*i7I|{9JGpd=V1>ec-mb*ADaZ*Xp`k|a|V}#47YHg)p3M#s^{A+bD;lYXw z-3T!N(WGdJI)WGF&-|hLoFYfWQIWsUcSB?02 z#?-4c&e_Df&UnAX3L~<1)OD)pwsxAfQJ=esJ<-iorx>{MO5 z+pr{oS-Zj*%(0E~+*Sd%LX9!g)H9yhVVxyojnpf`?id9&)E!9y1L(Aw0KxN;hEL(G zQh(RrSYK32MjfOpV+=#Pd%FIGS6|>~qsX}Ar}-}#I0`-iu>rpmV^J0Pv1V(*?{y}& zX0eQ&_`>DR)6apyjFDbTeaDGYPvVr;qv#!W-T<%Vo#smm+MT#O?XE*=Os~9$QjN_6 zo?N>;PrBY^?RfIm5fc0KQxd<-(N9l?;$9n1AKc7f^-f9SnVWx{^hZtKrlw*CZ&NZ% zxBBR64&R1Wg!Q1U0~FG6US-!{#?mMu8t`Pdz%7QXRsaGb2tfOC$uMTV5gp&4nD)ch zN;fFC4j{8kJ*Qq|{}IzHygoBgcBAF>DGBlrHV^f-m*#h3r`tE2fVr61XUQ*CLB*N?uV%ap^5Wq` zQ+4jyP8=DU!KQuAMme5vV?6CSPcALfv%l5hc=h3y-fy)7yNyOw`}mM&e;X0hi-Svg z>Gd{2+rXe8Wl{BSBlz;MIW@QK;I#n>HM&jq-$u^X3+|-YyLaYq zga6Kx>i;U9dhx#+=l|xD^Uh!3CQagNWE&}W5T`y-?reltP-qpdn;}KDo$+Gm>?&Zm zxLF8*-s)9Iz(1;WHdmO|%@W=`n#lGt;LR?ldb!uF zgxbTZ-%mJrBpwkvUs4sd&N8ib&;BqM(x_I-=+KVr2&z$znfo$Vj3A9gsv>gExb6a* zObY)I%=l^4A#6qMAFBK$yL`hZIz~@7E-r8+3_U?R+*@jB8!*ZTpu-COjJLRW zPdgJJKaRE1BE^xp2a&SZOPrhP&4Xf6r1+)_i&UHYo^Y%9b5J_2cBC`Tnk~N@6T7G0 zai*lS5?jL2R;SuZuY1(=LGx}%;HEgL+4_o!{&hP~&S{%&v(=-Y-Xo{$z=5|PHMJiT zt%_V1H#O+-kUkU*2+ZkRGS+Q5c}tU4aCp!X-gJ4I_G_jZQ1ZeYIkw>@JkULd8ylH0 z^orBaGa0d{J(TL;F}t76l<*zsnwI4VlY2*9I^=g$vnCD1Rfi%e3k%8)@p zog-;MDNjyON52Ta%f#N0v)}~>tUe4RjBx-5-sa1as{?E6rv+O`WAbf%#RF}sMEM7$ zHpRD-N*~l;|Hc-Zv^jhtSj~lk@dg$BaeA-?TAo^d#w0-aOx_2;h+rGMQR+ zuU@ls&7shfcHRrUvi4YMJ@r!I?e2!s{mv3zcH!bY>B`;I4rq-o_XC5aA<1v%2069Q z@!L@j+D*G;)R;g@^y`g2H&L043&S1`r;TvdXOs9nr8I`WIu+rZj~&E5@s)*(lP*tO zfB%H60B}?RGyi`^xBU9*ANAvJ`h~)|@=t8Zzw-b0)0t~ICb+9vp}A4xPMMpo>fI`l z!8cGLnm@w$1x#Y32!<2(7Em|ctD!mCfXZxX)Ms7xSj*Gg1?_6#A2m+IV%a4bWT4*G z_OaNzU(-_%oSzJPXJ*AT>_93HMl?zoMVNTwUnR4$J>&({!=>+tUK!GtNz9-{1tjjK zpKQKxDdFk5Of=M`j~z^Ajs`W~rckA<9``pB?|x17af0khqmmxIyyTSfM9i50^cAwFX6-%xE;cLBJ~|6gRM2dDIuNE#ZYX zCHNR{=EgQGr+iD{8)x8^M`1CpN{Kx$;l=)oE*NLFeNN>EL$-E$C%bN(yF^QSnL)J6 zY_)ew`z(fVGQnq^eMK<%mTsdF6&y8?VfwhF01bYDS#7Px89G*xJxin#>v}`WiQO zQb0%Yu+f_rm#n3N+Y>1a)2WabganPi!|8v_<<`C$dz{~5dY?Pq>cc7DOC6dYHH1$E zIC}N@&PqM5ywA<*tM7i|lqD;qL`!gPOIWj7#&K04Axu%W{*fk27AwEZ%p`lS%|Yur z=m#O^yXLw$OD=toeY#&KO}UsS#{}}(<}j0rfxDt${UgYDEEYT^rgMFjHTw}rW6$}M z;rHJ^z6LGvRt4~CCQEU3NoCHaF|#EZ;(CSfn5)Lo5|@j)Z=#xhUDw}UVMY|+A8e@G zb?Sv#FLMVZ(Hwhel)#Ch!JEDoeY`D0mvLW#7%V%I2LGMXhI$r=X6-kd+2aL|8CW;| zuSMhh4t4r}D?sS~kt$sQZY;~g@sR0XoWG5X$$ztW9)7LyD?X2{>8(B^(__X9QC=ff zr%OeEr?qbgukyMW3A}!ZWY`xm6N3$LjFqe6s=2($+I92GVMX`%FBnacPjY@@KWco8 z=`rZ}HKRj!PhZXkCKCz{zh=DoyUc6hz?LUZvzcs<#{WVlkyAL{oTwrLQThzOI67=6 zy4ubhzv1aEzIJ+IvgmEbw*SHWF=JF`2GwOk9k;sSo$UFj#eIrn@kxYVXh)*?^+9~r zIc``*M*e!@F*!14s@6_&Zu%-jbOPNaek$-~@%-Jz9>lT0M%`Mz(%7Ihr}9F<{;N`0 zQ#CzQJtcz_KujjRVIoF?a<+8k7qJ3qDs@E-voqNh19p2zHC^q+K90?fu-wYB3w`Q748a+ zEwe7Ax{1)`t!Xa=@#BJ9dB(;N`C4wb>(obN_S~R96#kKH0#<8c?>kxuk@EIj&;C1>Iz7r$t!K3 zyioqRw{iJi*Vf%F@K%n*;uAP5;ukI12{fb+-xtjJ z?UZ&1;FAMw1Q%H^v-v-ThZ!4;IGg(Vc(U`B<9k@Gl&Aa~a3%pip1m8h%BH8~_OHos zr5W$w;Xe#WT4luZ1DTeFFawHJ4V(23*ys9(x#dL7#o})Wf|+H5cAop(jN8h-{6W{g z@9Hk{awv!<5bS+nQK`tM*2-XslPhbrT4s6ESYd0jzP0`Rg7B1^wPIWH``LKJ-X@An zI&Iuu1Dw7mB8-v4MeQ`peQf5PI^c2k!n(N+F%07u`p$CfoNyYI{gO_ zq_T4(D^_WxeA01RMEUf2+ycUVEhWRUF=PprH)5ppAis9Y_iU zt5c0g!HSu%l|&*dbw zr`WUULsvaTc!V42rH(s|TqW?rGO>C(L@X9EgtPXTWT~wHOPM)=&6cNVwFW>q0m>px zHyjz2R)efpj)4Q-sv9>AV^?C)(^#T3{FV5OL+QT4A$y?L$y1>3ZO*f z5@ADlO@Y%~8{*%02ai73`^j*8{m7ug)5g>E3XY15TTs^enDLWF=#y<}d31x(E9oA( z^Bwv3v!XP5)6KqVR^qIQyvM;bWK~}YF4Wue0seC(1}0%NyGxPqLMXaMy7M1iOLV)A zCoa|9a45hJ47hEHa1ouu&bOVN*dd2~T#(FtTm0xqkzQ0r|%imVsWRs}koRmyC=v4zQ9IPLKs*C&A)6GJ9$< zHx4abpsB$MY=##%`eY!w06Ud30uX79#;Y4SYFSt=D7I@pv^v{uAy7F}Z?YS^#Vaus zK|(AA#-d@XdBqqe*+q&hsI^+%5F)&aD3aUNd@m~2)TgFLo;Y9L>!uKD)Q>A1UZVC( z7_VHQ#I&l};V+02!BWxbn-KZo35{^skw>@Q!Mh;VgFAbaL1J=_(#`?d8g1U6frZ?f zJ_}9LQ7KZZ4~$b^l^;L`tRIVbXQ_+?H?J9l1*Prw=M~llvQ-F`p21{bVIT*PSe-Pd z`XJfrcqEefNF*5yN7bt;(ku3&a=6YJVN_AJu+Zo6L0Wh>C`LOInv`+U^dmNPNu>=4Ir9 zHK&nofgbH%%^XmaM@2V0c5l$}uZ0hi)?EFwT%4^(4F}6Vh&ZuuAnk<<-Fe5Z^HdHk zU5&LevV|M2DPMo z5+%;H3!zHlwB{$mRwL9UV58$3@-`{@i>Tet2r?7=4lz?f=*tY)O?nW z00e)KJZX)n(L7a4lr+WXm=bxFl>eKTW# zj>4HoS%cKoDz~|gI~B?30HmLFc=`ikK?9N#qQz>pgX2d_kdEh`HDdVV&p#aisxTcY z{K!2B`v~I%w+fNVd8g!Bk*-CL#M?q=&@fn90=?2Vaxqm;Oq1QbNMmrPfX^~3ehv)!^jZcaKNj5>}a*Sl;nKin8q$3vB3=IRhNKDG*4V1F;Djc%b1Iegzv zgfHQY+O%ZSzN)ApZNRd(#=oJkc1;z;Dz}U_T zk~Vf|p+`z*CfD_9UyRM1S9jZ}eqeW4;nI-rwH$Bjkuoeu)QPOE>vjvLe`?JsyoTh_gCfAM#71HXJ1TF2~*EY`Dl zMkSQ{t3oDkz`uXCLTvA&YTG$*9#c+0RlSx136H-%!0^Z-yT$1NDN~C z@C>s(6JI3D;hs6E(eJYO<+O@0!csDaZ+v|WI38!h$%fqu@Wv@>hq|P>S6qrqND_M?T}Ci*#4h7$}7is^shkPhN4eli^X)p4); zO%?JF&X7NiQ#Uz%cz-Pz|6~&Rm(Bl7GV&j65-d9UxKKhk@-gt-9bya&>X#+E_^Gvh z!bBc1b+;!k9Bccd#lt#5;s_VE?!jKJyu%p*nL{tR%}9q!E2F+hJw7i%m2Ao_jfl@= zc)O(#z~<|*A>G9FlR-Tq-q`+(fXbmT?mM{Q)IJN(ryj@oTi~Z9ZN=Uzs{Gi0GFmWZ z`;~807$i{;Zt^C@KevTy`0hrZtZMj;K6cgcpl7AH&$)`DeQ`cfC3;+yp8^z#y$39l z;a?we1X=eq(Hp<4hf;J;rCtBro0^cyK|?8AdD$r&q&K4ZrRI69&IOY@V$rl(&R-V> zT-slpLly2Ps`8TcxjHDtk~!hUa~v+K!tU3c%mXgx#@8=udk6i6eJZ5f_T$h?{56Md zu$X;BglC0MXwnB`3a!E+MBB~U79R0fY09&dbkC++eeuy|)D9<^U^`=CM2s=`?p67T zq&-8VW!fBeph}*v!9G&QZFwES#kL@lyx}HOX!3phPS<^QV-=cu$YKrO`HlOlJB_Adv`kQl0_wO3C)-fZI(kU%#*cmkBGp#jZ5hH!aK#X6lQVHJYJH^$od6*B2E3q>&akB`8On8H+>wEh zf!+3oRNO5vsw2laqrNo!qdM$I*Oi|Pvql_?637_25tLIU=O7K=ho|5{Mmk!-JB^`L z3CS3H6KRxa&ADih2-PzUVHnWTSgAcU=D9i51r00C0!L1ij|{LY>KEam^@_!buK7tr z4w>KFRqrZ~a~Y{Zu=+^|tMO)R{oIKG$fR6+tx!dnD}w5wqq$s66Z#l`DcQ0YtCl6X2tX<7cd5~XmIb9V9>z2WN9 zV}hzKk5~a{? z+pc-#M~G=?ZrKxkrC@H;ih&tFWsnwDG;l2Q!2|4tgAW}b(wVHduX0!8!d%=|I0zkEir6HS>~coBlfb3^#wjZhtQhEx>n~Jn#B%>QA$rr^zVkH5>e!UmwkV7Kv7UB3MC*DQCz zs>ZSV&k+Y}Y9&1yeGDa=3;gg2k3nKfMIbb%z@o#VA7n_n<%n`a4Vd-$BvcTNzi7`( zN*cPmK#IV?rgX=(XO()wL@9D#@SVTg7U(B96dxpg1XNIX#^}W(n4u{*qNs>D(Lu?`h%Kr1`xR{eTieXnjBz|e z4PY|F(11MX+!Ia~f0&;IZ38BsM}n$y7f!E`bnF${7m94O9-)9+xvYwbnJDA9y;#F=SObp%4 z{J#Vzdl4rYbo_Htyl*}C$Lcqj5;Ewff}6i%PERpWadllzep zEo>BK{lH1JtGceBVidCTiyoN#5wF36IZLRa>l*OV0!8~V9i%!b z+2D)_Ka%=a9c-0F*vsoVSQ9fEzmfeG6m?nBTCB~N8O%2LE_@RoC?~hbVuEuKVCQ7{#P^pBJ*Op zCJvlF3brIPhMi4|dW$6(uLj(6!&`w=E*NBKWK^b4#B^T$c2iG@B1vE+aZV%LMJZAb zCG)`-ed^}sBb3w}B*nF??YATyt{&GES+~9{&l)16>gg>LtsScHMrp{CUxwZP8zy1< z`9~iB4RWMt;eDq_-=?CCURGVTO-CmSb}%m09hmCpsfTFk_+5fOA3Z`c&OOjY867}j zO3x2E(}R}4cbK=kkkIr*2j-SX2B)+I8%ewJY2+4EUfPDrxmXe*&}h(F%K3bGDw6zl z1ghuKiSTUc-*ed{Tu>_MOifhwIe^fl-ImsALz`vFkQw5@*(89gp@TkpFcj`=1x|7% z5_bV~Y@tGsW!PtdmkvS(P>bBusm7~AD2EcAvXReuG2P4T7`b?YIE#+o8rtBvxZ)>AsJaOL$P8HlH@GnAs+^nX?426ssYL2LtY|jvfy2{ zD_ausQm4OI98dZk*a{S!o!ER4ZB=you?94M+K5rt^39bm(V~&VN3KISg%VAf);!T# zL9fm|{#}RnRk#LmuBa&O>M8E|Qgac+lYBDSNmqEJE%VaTJN7L{F3kUgI+EoMjtWH1CuC6bm8y3KHoF$=MIj=zU8RV{lMasnX)Pfx2alWmx=s* z(|hgiE{E#%Z@hfJxG4da_pglDpxBh;@Rs8W<&B>48hRr`8h$10`Zp1zqRLT5mxI{`F=8(hArw;mOMJq zNKLwt-*9XP?9bir2H%tXWHJ=tJVfFF<`PxcJ+Nn%NKX=7-}!)aRHHu0fhb=jo{5^h zr6{PXjc|kVZXq0)SivBH>tg3qD^Pz9xBq@V7>;@sq`WXTTtBQHV=uEWmEJq?codNW z#tQcptiw<7hNcE!6z1)P%gd%l$1IBrizBCtZ**MrdAwlUUH?4AgWlGTy)Gb8(^ma^;}N_nam6h7!}2L%q*3x6&rf2dXJ{Lsast1X7v0}xFPH)+dg)RuD*0pmca@R}0p`9ng^jD% z=K{Q^f-F`J4L^H{8ea(84lUOftzA2{A6I_dhlSY+z)j6_@;er-u9Z-K%-ong?t4ZK zA5iG_tp-K0nSC&7?Zhij=REV{lPUub`V{5sw;^QK_ckLpQQ05Uq>&@J#}$=^&M$eJ zhmP5GV^;&MO>K4$4LnOQtlygpEjLXK+NJ!CP$G5D)Hv&VlTY@-fYVO~Gr~@!(!ha) z$oFieKfKT3efF&eZFX5}{!tr|-&W-2-?d4@nal-?ruF=oHikC>H#=L2!JA67X$G$O zB3id@tG1|d^+>_r=QE)FzTD$xECxouD`0F?*Rv0(!8o!M@Qf=}f{@h!soCn4-?yp! z_|1ECKQJ0(kiQ9c=DmRmD1*}nomuKFLz7QkP73nh-F>WkQt;z{OVWHow6(m;KG4hc z)0oaxttFvP$5oXBDyI!B^d@|G!e%2b1za96)I5b)X!$U!i>R0-U0#K8DAJQRcaz&j z0@jr{wIrLo>}JDauDIquv@q~=92U|2P<`vt{qjN9fZIj@FUhY|p)1@%fK>K&3R4d= zU30E}Rm-Mx`IA`MqH}TfhRa8`CHx`$bG{jScaC_oKAl7@g~QX?zm7Yk^~Jo?T#Tez z@-Uy+Z_F@`OPD!4IHeQ^bh(joP1CRfeh+L;EHm$QH80gJx?0Kk+gYXGnD>o+oR>}Kwyp}C&L$liw! zgG8)Boj@89H|U;g&kJzv+byvNZmD7td^t~bhGWq!HZSs!1G0-VdC{gM;^dtL(8r;a zM+tkSWnbTrfF}cc8*07Vv#OYQUsJv>pI5I>3nM!$qkh~k9W+skr|6fEGOMM&GgYYEoS+V} z;i6L~$}FT4U+35wLsX>itvGqD(@u}1(eCdj!N=fx+R8{f^N##GId(RD7k#`3r?aT6 zLv|}X;}%RS{;vp3Kh`x$Fhng_+l(i|tFToj;5!67$?4D2L-!Q+VhcmA z+CMFvFG~ziRTAG3%CSH1&!%gSj=F08;Bsya=j_w$e65=Wp!ioJOb;F~Jx#rE_`j3E z{uhwokdCj2h5Yklf`R%eJ~xS=8%HHYsv5$%YLm zTB2b-y+onpKSk?7uU6K0C9nyZuzgR0aNiM(K1U6B31WYyEUa61=erF+Zp!@Xh&i1e zLg6dMea1Xw7}D0uZ%nG~wVic%d<|e7|75e+_ufwU&=b+~x=p!n^6Qu%AF=)nYMc|_ z4!&Xk2srw%oXqs*GZC~bW8c}wz(e>evAHuCakzo%P9EahsC`_=y2jn~>R;Dc*F>vM zz2#f~y;I>$YbK$D?B53ii$gd6>L&l4@&EHHmh^`4(~|*fd&Li@AKPudTDT~6D6U&0 zIpc`^uSS z-9Y88hifC(#7E?w_^#JgI*u@owrz}K968;&JA7lZt?JzWr#68&L97KSR~Wlcn%F*o zMC5RSx#rFDrfl-BWyR{^YaB68Ib^_;TOZ}z6#+1diz6yZ%5}xkynHyaAWxiw8q0b~ zJ@41N|Gy^23p^hME)hOHiG!CurFXhR!b9R;5@<;mzj*G#W8g@B6Q@hOTW$dJ+Ur`X z-3ni@o;g7CC^CqfZ)I6E^|a{}0Q9f484XPfhJ0+sHIe*|n^knO;cEL-k;dG#@A*2m$LFar3aksYA?aD zcX#hcgcRojC#|*=n)Y}zfSb3spnE5-wzAYB39 zi=65n4AsMkvIC{PTgL+!51QpYLOTOm$Z5n~``$sP;+074yuiD`d8Sl>6@e$1B3eq6 zP#{)M4rJ8~jHaTTEVV?dTMY*mYA-zWA=xXLf5@pv9h_U$)moM1*Yeoql$BR2>gih* z0u@yV`<4V~ws7=ZyZt}GFVE3Fbsm)uycjq;Cd{|a6LVq`c}}H0UZ?1QMzah%33svq zt8G*$DOHI0nvsDtx2u*FhkD&ZvYLOkDS6F>e20%te_@*|%54WCry4eE34&k8Yx+u| z@3-T1z3Ao^7Y;i~QG~9$?q3jon#}+VNwM6AddX!+EC+}(RuT*PG?79R+NB1sg9K~w zx+`@SYKGsmzZnjEO`{W;WE~Ls`7~EV$4Syp1_M(5_esqFwa+DI3fvXSB3zu65pIXRx8TkzG|RdUufL5-c$EA2fbqtpoOR+t?Y z>sy>TcKw-+;T<=t*!Dh6Ro;M0cBpdd*p*C{$nNA1c@IQR`k!sfJb8c5UR1*eE*UBD zmc3_u$*CyaabVmrs8*oN<1U%1*Tll2bY$afFMs?v(CLe~s&=PBp(FTY>$a9|UyZL4 zSx54yY4S_ESh%QR_i1{dvwj;(R8n2!MlTQ9N>uHbW;nVd)5C`)0SJc1eqt1U?*bDP zuK8~}r2pKI|7si59XW1LYXp)T<{TeaPqLnm4Oz3uIaxj*D-i8pZ{Z&8@HNvAzm7!= zB!6stS=D;g`nxA_los`Ec^a|mzA`&H7I!j*b8Kd*6yzBe>Y8FyS<}2aIidyO9`$d` zk2_>(YcpyeI>>qI+c2$?>KoWqFZO7AUGDm{vQOiS2BD6q=vCFqb%~!0=3_zb(=y`Q zr>2#Cfh||g4XupsIZr=o|4~-8GuaYaw5nPmBffQ@5I!!M-ykqV6A}ZFUTz+q(T#IW zm3!oDZe*VRHRe>Xxor@%&@4eNKEd%zEX$0VOU{L!b=7BDGy$ZRxXNrxtC+6WFBbMl zjCgvph=$d{;PI!SJ1NslZ*aDG2I=~Uu>qak0MCdy`R~*XwMRHU`#Wz{6lF>Se=-1r zggxn>42GkYDxc9s7aLd5cYiXhy__}a;gm^VsSmMj#5r%EH(c>wn1e#WRXM@-<~2_A zan+K-ni?;?#g6mALB->#-OD2aRB@lQ?s;^m8JvJUpO03lsAvq;81{b`C9}sbYH5}2 z$@X37IXRLxW8Q30&qWg9b_4BCuV?sKZASB%zE2CDh#`t!-5@Hhm*(SptMWFPXr*Hg zoU{Evtj$r`F?qb@61tk|g`0Yv!Vl@~g2JK`YqxIByR4nCa#r z>Z(SdVw|qB%XKZD z6ZPhMnmyK$Hh1)wt6yLWQzC&NbQ|133lg~oV&W|X0HR(A4y@YNoi!guN1XcbvDrR? zcbx+%d4^j5a91CG65pL0=W(TJ~pXE&ZdnjQDLj^b_l`s`D+`@-~#Kt7H4S9Ta6 z(p^n6JACt2xB047*G=$=f!iJZAO1%^6^C3VSXo&gXLwntq&^4qK{hpM#mVkN z$xgzznP@T?wYtFDQ=L;%h7Pcw?Vbk1!nSrL&jw$uSI~4g{?Kd}{*V$NL#=q!$t!^i zW{$ZU!^^dfa(N?;!BOsZZOe=W3aAZ|1;f9%30v!rBsP^$h#dK=oe6L;T99;+pYJkB zz1>DXSm_i`kqkWUZ4y4W%U_xhHg~k{MB$`<&*PH+Z>M$T>Re2>+?q12YHs|p3*RNe zua~5qrIbY`+ z8mA3hmWbU|=sf4En-V%L>11R|!zcY|bz<{&FMhwh?KSYAc|SYj?szNv&n?&YDnETa zOL$@6A>cZzU2E$9GuRh?&s&(W`rxvDwa4|7t#y;OdUtu=j`d!%oZ}g=(9rntpP{t< zJ>SOv4E{4aSKkfavD|HL)}%QZ$5T!9c85=VSuU^qYuD;7QC)SJ?^7cWf4HfC=EaWm zCy~28tXv(tl;zXQ+slB*S?`P8|H$(Df~k7WPwJaq7s?Ax({=`MLPjsrNZNak3Y4zGQQS&8pp_5wL8RllI5_-F4ru+`PJYC!6Z>x#9DUZd~;# zCpBgEn=mtN+ump$*1(z5&KJM&ec^p#Uhl4Mg=upw9&c1ywx@f+hVG8lAwj^y9*_XA z`_G~6T%@$zTgd3vi9<{iB3uu^rk?v&o$=)I-XL>rrB|N6q{-FXS0~HT)8nS3-f1$s za=(eWb#-d&x09w*=VqK==Xd+U%l$0MnST14bQe$Ay>->Y%WA9EPAz{{CGvLV)|-OA zM9m+x_S<}s?yrmrGyPtA@A7sX1-%Z<{uh{h&Nepwnz?MSPe@3tmu$)~bOmtStZDHeQr zp=3Ho#AKiDkZHbM%1eEnEL@hr!ba@;#x;B9Os+7S6zQg$#g&jHTDjh6^6p7yMw{bY zj5nTd3~YWm>HF01Nk&uKcT}Lh?$lycVcC1$YQ9bcZr@0{h3R(!k7 z$7;J#cUrD?`Sw*htJCK*@s_Ml+2g&qQY~JaefLIFYp^jV#hu>yPX5KdW0zUP2PVsy zEgg670Ot*S7Tvg6-C_7w%ki$~Ouv(rtf}vqh0KF|PV^pGDzs_SccDvL54~X%dF|^_ z6*9?qX~z7<Ry*=yNdQLC? zonOS+U)FUyN%mO@yr?ntx}JX~o_W)j?I8;$x!Y{own)bVP62mlGOb-cF)-rZ7ftO)rrAw>*Hy)T6@y183_}-A&CTDs zheAei z$MhChjqYxHpuJTT#ju#bP?2b$^c%&kV(+9Ic4uit0(lPZv?9Jms{|MNG|IeOSt0Rx zTlWVy;Ek#heCOm&`a8F80uIdB^2wd}Vo|MD!OVQ&;hTPRLDtB_H&t5KgnwGBDZYdy V5*Dwfep+(wTym*TzSaGKF8M* zsHq?GJZ9zK1ONfF%)C!|xt_9f0l0o6!M}U=F7Z8L3SwdkE;?#DuK)7!>k}T?y<4LM zc(?Ic@NSXe-zLNV)rrUWn@s}z-~9e_+`+$f8^`HAB3v`{A>J+g+qdrA!f}547CtfI zZ9H7-9WnxPLW(B>FJP2Za1E2VyVNX#i8*7BJ_15q8#@_3-mP1={^I68_`xT;MSlAU#T`l>6^CNe7-qiZ4RIf5 ziYu3X*)qO9ResjiW>qG0m*){pv_9#u}IEsqOT`m!m8)= zJ_yD;1%AhSxDGor$7m>9WN^}kroJHdd>89Y{rQ4CnKCjTg?>Y6Q&7dIk>Uccx^>^T zxBcAW;!>cnN$`t^wY3?We&qWJL*1CA6A>l~x*@$eZ-`W%eCoNTV#*1&jm|FMhXRliki`-Ms1Frf{wxH%BY2MMj&B<;?{aGe z!G*7mcPo|jFV;*{2L?$_1ofH{jg8_Pc@*eB(FzKhsj)wA!I)Zn(!j%`?Cqt(!y_QE z5dO1;3}`6FUn~eKr@b(?A|MV7`tqaBJC+{g7%(*gZM=y(b$E&iV_n>NeX-qlbfSMv z^B}*jkxoBN`+WJIdar%0YsX zR74#_^mnlJ*3WKlx;Vcco8l}8LriZ&^W)=~p=(6LO@paLE%ok2eeX*S8a11;<-{A# z(NgI?!XmZAfc}@@3}uGehV^s0(`X)rYVa3L$7TtWo#U*zjLk*EFMfSIN;|?NXf2{T zZ&l!=Sjonrm%~rtT@;5E*2k;oL6l3Q8{I)iu%zIFFi*hbUT@R+nU(5l+QEa1r^S;Q z0&BnU(h)LEKOCAv@~p3zES!rq&;e81QJLMPzQlEXbKz5RX%h zn6#z3Gcc%JZJ4Y&)=V((-qSy2yCqt9Qkl&N92-8parO9c^L}Ru1EGCv1J=*FHwDW& zPtb_T6-gdUyJ-arqe2H`fnnbnVk+r|i|6AXWF=#0uM$f!-LjnACUi>@f=i#vmLEz= z`qW69BhP(oFXKLIBzax*g$%ELB$FRqEk?Xjt}={^1FJ}eDJG&Uf<#L18EEP~LMR*B=f5{3_ImDfF>5xO&PODc+!Ee(|@6BeMc z#z5Ezfi6>#_YR@|k9)OciA?nTUn<6I;xo*T--*U!pHNM{$B}#H%@DolGqW2{$?3El z{>rlwyZez6lH*F3nYta9`PsH{Y$pjQ?a<2z_f~)a{KrrJ%kS%dbRZUF-|PYM#xI+x zm2m8~;8X4G33V$|p6iHQ#&s`#E-E}uxRvSkEjezv6r6snFp7P=?B-NcG=1F15u&%y zwV?jBa1x;lGDD!i(6j}6ZBrl$=Y!d|NU|F~VVC!8-uNpey7wxtJEPZt?gprOPAMo2 zSxUGyQ=FP+!SoVH$3|o0IMI8p*!kX7%G{FHR{lJtjA*pGNaA2WC{hUn7}pg&>Z8iM z;Nd&sF;;9U!*r!Zq~#*Il_jf^d)_3L(q&99VCAHwxN9}}@;6Rg(}XSK`^My}*ovyW zKy9tVvlKQzrmP)KTi?Fy*OrK@K=Y0X*Tp#~Ex>dl$OT#Ulgl=hisO`9SPFaeE69os zxGF)b%4s)Uyo&^B$%m+X$Hym;Q5wFqN+p_BwRH2v$q5}ChE}%1lU1`Obo29RW$4+y zx-m3d*`Y5qIv2xJ<50>wDPHn~MMH{)QbDPN=9zPrW4G}9;^U3EXqXv(K z-mKdZ`;60|wWl?1T3iN)iVrib@ZF=I<7`9Y3b+9cT=ZEg@v>#Y=tqSD5tP``9 zv8~rysKMb06YCF>u4>jxK0HFRG~8b0ca+Z?mGd84dn}|`8)pNp^LQTTHqjU#ZzfOJ zce0#lflpwZw+S_gwfe58qMML+T1fF}-bJn|zl;m~6F|vU75OLKmty;$cXV`CFT_+i z5T?9ur_MZB;6%n4f|AFd@opY%R7I84eh@i7&{)58z*T;>8x@enQdCQo>T3l)9w;O92g-yZi)Pa_o9a#=0t6`^P;g@)9-$` z%41>pxEbG$yi(8)tmD&&Rs4J(6`4dxO;t2s?~KWVKjGZ>8veU)H{KoF2w<*tuJUu? zgz;5?)xFA{qYGHuXFxDvxb;Kiy}tW=fNjFmN_dlQ$G(u&z*coCfXAxVlyD1HSy&!Hg3o* zQpA*GrPD~SENpzJq@J3E8;1J_J_k<0$ZM+-_91wn9ET5JL$Vj|fXyk!`H**a;1I3` zeDYEP&KdZJ+#B+y{}5O}BXr|sW|3Xi(?26YL4g3CC!@HChkBznf1u(ip%_$F$z?iT z(Ac<+M;-F1(TzJWh!6xK#4|THHvVrE^5P606}3Em@XD2p8nK6P%1Z0zYc-4Y4u-b6xvfON!fn5>Rl`X!*B2eC;V1G(jSnn5BXPPxF&9!6~7p=>Q zk#IpzM3QW9)OFm2N=XKb(;o#5qLvq%j-RrGE*HuOc z78^g}GW?wOWsPvErcQ*r^2ACowzR70dA* zvSBs{DctTE%qwRWO7aQ*J`;Vm$v&CT+cs`!T2#5vAywFQZdn!L4UQ{rYrxR9m{(^C zg`wxHv7fj2J4?z~TA-4dKa{Sj+0Kb-*VI_iHRiV zW=F;cCB^i(JIf`^8i%Kw<0;EF+1CtbE>5+Uo8O`rd|vtu+q7aQj#>MEIv<-aHk?)S zRW5XSg}5>c#rj0e9}cYhB8FmFk;8_;wp~{#r_@pzD3V>PV$NcWxOruIGBV4g$vwA4 zQBu}QK=0Z)fV~1I9_YYVx7b~J=UXC9#I4M;<2BRYGA_hI1os+`PZ_ShWuVa`YcjI0 z$-QajAjLQ+*Yz!zSoO4167}?BvtD0icfYG|$<6Pr8V8!ctv9odsAu_BZs?pP@O2p| z;()WZS?_}a>_!~DC5HZBQOket{w`Tt?u068;X{>*Va&S|3o7A4yplivZ3zDNDnvu7 zbQM0$4^AQVn}Wd}szP3Csj6@5gwO5iUs@T@@4NBcL-Ue=U#%KX2kPp4Wc9AR|5j() z3SGaV&nWSYk(u{_9yLM%+K!%?nVa8l7kNPiQ_?G@3f8#w@BIAxOHyG_U%4{62#~1G z&G6)TxTgn@UfE34pY6&p9pkFGp*MUcBWR%okqOs$#pOjVfQCYK`iZfZ9T;?DMbyU9 z{YCA!Y5nIHeJNa_;#K{!4|59gZ^-jZ>sCMqtUQh^mMn}e0}}EWRKd-oR?Qb`VcUJ|jIsLbi z-f^Wmh_*vPA3{h(^WY1vA-57n+sI+g$F0c1LB&-_dXu1-txuhOXtH)I` z0;~Ogj_znvbqS{0UAkQ2DZu*zC;GkBzNTK0Gd^sKGG7sdy^fP8)GsQ}2!5>C`RSY> zYh#Wsh6h#O+gF12r}av~HY!InM*N zCK{Z@lk<#~O)qIYMrkQ6u`;M^V`gBVC@$+rEud{lanB zWZzV+90t$Z_KlPa(bBbC+YL$A^BI8+OA(uN1elUbA$fS@@om?*1dZBzAIVi)`!TbD z!jzHy!2g4Tj5Vr!9BWkZ5Ooo3d9DqV^jmSrbCk zyP=3kc`0ko(XPHU%OBl@5^~J9C0ogcc+FWw$Yqp<209n*!%QQh=#Myx8EPhqGzHdR z6PRPMj0W~u%FT%YdBos2mj=?j*=wjQe-{?^sl@EwH);Xp%?8_IM3#?vTQFx#{?^L( z$lR4?W4O0uk!{gLc4yQs*0FODVs^pPCf?>3OeDpA+3z0=yMqZCxO00*g@#dyMbXz^5t7NDdlXMqa)92E#?Oc*nKao#TTT#`=8H5T{;HO+; zt>zdmGf|Ko7!M@!q81R`s>>|}T$y+D7cPglIlnmp^o7OK<+~!Zp@|n)amBE#!&I?( z3>02|x6yu|?(`QPS;;zE zQ}WVyVv3v}B?}VEX4B-nRIo5moAs5vZ0a>du3OF#U?41BD6hOBIf(icX|YfgIk@Dq zy72~mcSA%?rp6a%ZJdBIF7}3!w!I~kH1DukM(YSyM*v?irz7kc1sW%yK!4J+KQl!`NX1yW6ILJO9Dit+ynUC>?kAw zvnld2#YN5BtUt0qxno5nmQ+FCaf-*nan32twil>A@}d|q@>)gMpwVAROg%kCCA}A} z1K@u3@irp5(QM$+P~+7YCYHPsB{R@C%KhrwAD|dt3Q%hYK`>oJUxi`h69T4gd1uN~ ztKeJtmR_+58-Buw+)QwoA~N|*F(eH>jQ7Uc?j#*FC48(#2zE>KW#yB^ zAK9ZmN$?UG9``<$>>Gz4Mg>GOKbG*ch4GX3!HQajOXn4PE9Vq@bvD-Jic`Wx^i?}L zmy5;uZ?n|LNa6QlYYMZ&V-qx=QB)+A7zM^?sRs~L4kFN+A1PeXa@HjYIaX9FN@xYe zs>gk5mEJEaa62#mdi;BQT!bgMhlkgW)HfY6^i822D%EjNnbxXZn2pN3LNnL3c~+^E zqx#AzcuIUyJ`b@nwu%L@Hl_)}n-k2@rfK_dYVl+UJbe`mZK;d<0aQ-EYg4zY)4}AFg;a86Ewm;+dVY}assCXT_!*;8$QsfTJ zQD{GsR)QyRsGl8Ka~wyUL0vQP~K>0Ty#f;M0;28moeNv%Rdvfo6Q;G7r(Z0PBdx z1kDC9>?fiU5aipTI{vxEeFEH=8l^J^FB)!UvzMx*ol*&?6-HJjg^qx>l@>-w4P_9u9n&sR)`%jEvMQD?lkXq*u zeIq%_v?T)6$D&cP9b9p3P+^+6pTGegdxSDiXUvC^lHu%CxB#)X;~6)PRn&LVsTtsr zeYfq&lc?>&CiQaGNL~)*(ayLyK#0?uBgVI?&yD9IX55O{q}*c-lHki3Uo|n#Vc11MZq! zG?TxYEsST;8{pwaB{{IgQ7u6@KBLNDj58b6HBN0yRx0Dln=?FVFfYJ>$*MCPz$xM^qdOw)PI}oY*91%_55$dr>c% z>D+~ZcTjFdRaLIdZBZu5AIj?;7iKL4)nG9 zt|wf!nYGz}g0wj7M#_(7$4S{7vHJ20UM)*s@}2>_sjn#$n;%U4bcvHHSYL=Z+{kv3 zt4&_&me5tcqN(XoGEKTca`^i8a4y735sA$V$qmDf*z|S1c9YkRQ40~}`Pr^Aw#v#L ztk-v1g6GUo9<6)iNOtTw;j$~m#O8(}2`xYV+@N#a!)6CU;`&nAP3w7Omi{`I0D^;I zRuVJMp|*`QG1amS1q+@Fwu#z!!(hG5N7@kEW%0Hx_Cm2Wt=dG~oNzKKB~Z$SH74cS zEj!$wq*3CMCpxhus>W_E-izN_65(zwGxjov!j2GU!Mxn zPmJW&n5(iaT3XZTr8gf#=*k@^Uvh%HW1jclOX4%5nZ%jT=q&|)pmyZ3d?vFnp_B1; z&(H%mzfs!$M~!2r3KGYs&n0Bi{`^-*iYt?SxnRScwV^(%w4a!jJtMFUQ3V*M+LmIi>N5#D%7fA(-+fyR1jys?pzF zCsUKo1Qr?gS7EZLB=1}LTAI|*-TJ#miZX*(hjrHUF3@*n>cialLsozlAE`AK(AC2x zc?l#3*lp~dvn;_A90|94oI36`CB9wi@=CQ{>;^@TVY0~$YiZHbY*DBQJ|0tgq>H5_ z&NQSC1QimZQtgKYLdF}OHn*Yn-n?pu&*|=j&ldEtP0!`pMhl>Id2}%{dZ%D=GuhUS>jb?o#I*cCWPHF;JI~7Ho@Z zYnvH^K-7RigBdeviYNHaGqpYVNn(*~FuSercV!`7lTfO8({h!FQ?BV^?{BoMa$fs< z${lwm-N5x_1vZnk#v=|H%<#^EEFA^6Y-CeV-z}eD#38>JRObH6gCYs9sH)_KhRXK` z)jkoY0y4^+3qhqoKM1=8V$E`MPW)*iZ*4clsHEqcU_d2SXEX2*? z##`b3L&BxFkZ(jTc=~;K;y^6%Zi}`+yPC4@6GWb9WFvWBbIN@I)$Pvmd<7Oxpw3e* zw8)G>zDZP_YzpN~id5nlDq1oI$<{lb)lLiE?1^IcF>npKbY`H#z_Whgkt;~LI+F>i zF+Gjru8Cy#C1=#5=%aJ9@xw{J-V;?<(H7$yF9Q(I`2GsJxMs&So;BP^mY8~MMnut~ zoD{vGL}kCVq1&5wO2g*7%G)JXSe3+Q8f`)6iVOK#C2A*FHnW*T-sDmdY0nw7Dt=+ON@K1$ z9d7~LMC9*YA#kL80yFDVqM8a9)(!37MapK#9~`m@5@aVrfuQmB-q?2=ioL@G^pHOQ z-VeYhLiox!ShEdnoGbeTtEo6K+d#xIvx*arci?bpDtpsD7fmV>=flHoiv2$3DE%W6 zEtf3YeyS>DoOf=vX39+R9$Tihrn~Il@dXUHh`meLe8_80XPZ+O!XuC8I(-fVw3r8% z70I_j<7?y7_vK3qIAkEMqm4N4Fp?RV=sZ?i!$=7Bh8sKxs zPxk_{rKy{Dzbm(uT+#3fp^YDpw`3GFr{B;vz7TzmM%yPl+1q36B3_cyn11++zyG|j zPVXz>&N~4Al?c^y-UXFR@zr;aF7J0UVt(H~<>uqoUu|YF(kqz_7DSPb3~1)i`cay9 zXH*+@RiQAE(3Vt@vi4`UvdWR55iX1S#wQz7-R=Che&NaM?5p+D%6&L7pX&e2|EYd* zpm)xKG^d2gAgvbN)7wT3knQgIyGI8&D570=G^&NPk(Qf=>u2O2ELWou`%#VbAunnw zn*hlttQ(dKf+Wcz3M}zCG%FShVz)NmjID3T(8+@Qq9Q`(bf{|lBEXAYEFh4=i<(>? zNlPVMOg<79uYogR9&Tvl{q=<97>_wH*+7gv*-%~(1k%{Or>fBoRJ3aer=_JLoMmZt zDm7aR*QI5D(URxG<`*I5)7K^t1T+qO;b<4H{mz&jrN_269vW+>2UhcH7vb(EbfsB*ZDV_ztQ}` zQZ&W=Y2en#_r7MB}6%YH(nrhY;6C9B~85 z0tgMFKYy5mE#~`0bkF@wyxF{K=a|=HEP&9z!>Lde6)Jih8|a5o$oHvbwz z1<%cJ_DaQ%by^@cCzR=M0{p*O-GIl)q)FEop^7enM z$w>df;zEv9%dwg!RaS~phV{phNjIR;B896YSETWQYMwRJ+$GtkE!?Y0#oi&ms+4JnXcag7dxdboIam~`dnp4*D%)((&%vtiq5umh-uCZs6 zk{WR<|B*NPRc8F-MSOJJMRb0wJg&Gxa8X!2-Qe;Xi0U;1qWuz1WK9+CBO1Ri^5c^a) zHqsz!Gd7MB3`rz0S4oU?*u0pX?sFyFJmAcy;aC7XSLP?zL{P*ps6cWqwy>3Buf3*H zlLHsj?PcW5=kXNKY68pj7G+%I_SfXE^m3h(LK+jM0DM~o9uo^U@3-v{jDoPbo(7)c z9crQBzJ}fpz{cdTuW9KeDAx`6BM04UzPX@(@Qhr0;8I=3uJl#3ri9CEd&Jyx^XGqb z-Y9tcsjeuq#qS61yhV2p0Qw!CVN-!-?H9>?u;Fo0dFX%PP*yy60&6045jAC zkn>h98^pxC`GqH!eO7*C{`3q--v^JvzXWLxdq~rfpk%(%av)TXR;%sljsVCK z2`b6*reApN=*Ys9@R(L`L=yBDo_p%-e8&%8F1$0@=lG{+%AI!;A`UFm;vdx}MM!A( zC>rY4qG@1@Mk;xY_Nacx{UWap8$sl&N^052yJ601Pztd7E7c}nR zuvs{)j)-@+=B@D&&PzPGZVUYq$nW(={n}_P)zBu|TD_{Wtme9gEt6e~iovv6S`OAT z=Nm8$jl30B{3fhnE&bZhv5OAA;lr1nH~ukqAz)e%TIMNzZe+sX5Z0Yx+eKFpSg>;` zDUzhWM4MN#pxscLsak6RS9gxmm{z>-ZHtt-CT7AuU=3S%8%j z^ELnzHzi%19m}THlfDnV9`nF3a_-P8OqQ`qj#ZB>4z6X-r3Evj{dBw)miMMpoO_o$ z!X0uTh&ss5pRq}HcVOFnb*XhUk&lr~oG5)(sl`w4$>VNa7Fd>I`Y9#t`&q=C zBc+_%H)<>=aq?+2KqIPNLWFl7sYNX`Ax^13`(opnVvA>|WI2ucgm`XgXu7*oapSAW zeeU^bRAp;fw6z$tX|9^b46Qjj>ko7e#pLYOUC?plnaQorK@aAy>(+6N zX1$dmhi*#V+Vo+1EE<}iI;KsAfpP;{4qRgGHaQoToi4{^OFJ(m31o=JL-gA6OEG(i z?9^!`=_qJ0GGM?x2O)9vHs_0=jetvnGJEoJAmS4Z(-*pO!efh%HStqXKjj`j`45ZMXPE1cFGp9=dXqVxo(0q8w$5Js^yMV~cT~f5nL?h+_a%2fZ zdK0>SeX{8ko$!vq>KH5@-XGHSWvxuVeVM_bX^W~MC%dB>Q~xi#J3bSspI!W=y-fAX zA+f3PA`^zgHWY=MxFrCIgFROEfzT(_nS*9)W@o+M(jrsUWbU~Z!@T2NZJF++D3z^! zSBY2^S=f4o#c;0KSv$^SE0YTR$RaHdafxPE)2g1r3Cqo$BdSC4g|n$^5q2n7%A2I< zL8I`I1r1DdXWN{+&@`)M3z^n()24|Yj#(IRHDE}hG(kztT)17J!F`l>eiW8of z67J$I%i(F;e*Y4sI9P{mF5tOw!YQkapdM?Wr>HqDRZ%~c&rC3=L zq%qXf8OhOBofal#rR+`ZQQ_VljVcn*(LK^6s$gvb1&YL8JZ|0(1k7fRf^!M5m%x;`sLJ^I3Fks)e>%j4` z#;EVgK*I-?{WXd4eJvr1{k^nSdIJ8s3b@3-SMJ=YpVbIIP6@?&+%+1BO?%MdH`Y@M zwNIrw2emxkGaU3A$V2jXZe(t%T*ccvY;9D(lPnxy@8p(@f$4HQ{+FrD?~x8m;!QE; z$y6Y(D9nG+lAx9Yk?3ptVAn!Qbwiss)A7GcLBOMR$HOzrKkJ16JL;Jv)ILM%0_nEqq>Hl02br>upH6=q0%pE;Pi^pw67o61JN)+K)84a}--TZL zd+U{#U}l%5&(d2ZqtA8p3^eV<0OM-+>ks;AJG`=|)umr1Y z!8n3Xp6WytKdNmeDe#eY6Lo8VsOEHK>y#Q!ww{%)+iPV4Pon8YBW&n*FC3(@m;*=g z&EnR|^A0G&YhUeY|I7sM&R;WHS$o2R8gi1qxl`&bGSrWK%P8#`<85^is4NDrU5_YK zi~_7ZYD>PNL0dx6x^)?OQC**Pybr%ia`=4Wjn<0nxONEdNEGJfTh!mJv6g&_qF!P- zO-Y%oo45Z~oxYl#C&L+LqtDF?F)Fr@%fNs>ZzGDeCO^ZRo!#{3rr3@um+!g|nMI7Y zMZ57*A5xXt>?K)`qTnH=tzGBf8v7?A;h;qNP9L}9&zyQrveCBI9UfTB)TYyoLm#-n zIn5z^?xJllbt2F#YV{Xhouttdg zp|mEWCz-Tp2wK{m_hGcwT^z>A-#Czhu>HbNzj57$)*^^83-e+It=pdk%j_l^6u&_jMledz)6dewhyCqy4G|Ir6n0ZQS+jp zl-6>YT!n}rSQPxTvWHT`|4B;X|F!4G&zrom9MG9|TZF(fZQt(BX13<%Hh>_( zSi7m#CF?_ER99{*<~t1~$CmBsC8tYH9b=uPB9;1F*!8=sVq^#BcIK?5Ii`HlKjG+{ z7K!eTq9-o3uK?V_--!}wt|@=vpjS3KtyRi|jM;x!Sj$2yUeQ4k0Uz6m|B&pW3a(lG{9Bj+HvP z-LiMSlalB7#fNZfd?zu??J*4?Bw$jurHUUh{7)awyl zJ400nbe4Ul3YBPnLr+Rd{ow?nS7NqdL@!=`D}ay;_y>oQY#1}k)m)bs`B5l|@&w z5)2MGf#~lDjF{r$_82LOqo*M+Hv7uX9(|)NoaYim5|X?;QNd6Jtq^bhZJIesAa4IY zMWY_~`=dJ5fy&X^1p0#abhfr)w0xmcl*gpBW_Kcw2y+j6-Y?5*8bQ2wVoJLN``~Jy z9_Ym+o=#Gj6tFeZ_l`q8eKfuO{H$xk2m+mFJ8~3#Qi~t7l-wEjm_6vj-=zMz63ccy zC$ohuagC|7tr*|hiK4n8zjuCvPi*A%gujK1eIi;NKSdpnkmT)Qq;QI8d}8XybY)N{N z_5>Q+$hM82M}S3TgAZG298u-U!v`(HTC#@$5k4V7}SzZ(>W!d zql~}+iGbfE!trF#37JqMDXoUAC!dmM9JvZLi&{_0??%WY|HDzY*&?WUZPHEoCIJz#) z^pFEl8lRYSPhfmhNf9*PG)37VIA{8%N|GT9(-RQr?oK4H=2V$zy&&i5%dL@2kpjH$ zpI^NmwYrkjX<)+3bfTYQd|*;}kh!K!WyWKvUXZX;eKTZr=cbH4qr9wGXU~&bgKw*7 z0gk%YI`J6_OVEw9W;;@t^pY^rz~({V<&O4+${OWh;g$J#W&exn`u(!(!e4ltuNENX zd#;sh1o?U|=4Rj+aOptVDokVZ&>;1yj~Z32xhDe@7aD_vZ4v2xmI zD$>o%>?b&T~x0Y zmqu%=H5aAQv_!wA)@qh>H7bk6Ln8By4Fs;j+d{!sD-*^S~Y&+J>r^VCiG7;J#+)7hPiS`Dp9~}jz3Rhzm1%kFeSC(4&mGd8RFE0*f zDjODd3%)5hRkOqS-|r^_vY4h~^`tGdvn%p^zA1_QUJpa+*9hF2fq$5y{bs9euXRlA zIUVp&AQ1(QR_Q5lO3Y6Qm2Bb3sLSp?PV(*dJg&3}$@r>Y*|dECvzB!6f_Q4qu6z-Z zlJtqmbhj|F1Z%kI0Nn z6UAfUIj>D1IuoS_g-G1Ks0cvYol*QVeTI2mdF_0?97kji3@bt+De}Y=gTzP*ElNRbO0$|PlV zzxi02C1qaYJ!(996^*>__P(hu9Y`V;h=W5`ntI{cg>iw92BV z+oLJeBJbAp2k7Fi+fBeznYK158YsEyqH#3icyI!Q9QI`*Z%@~vQ?D5779km$A{X6O zfR=PY8xCt=ts4*G=h~(fX4f0D!bnBO6bGfyBwUXVtQ%8sg!lkEQXueF9tA+HsRt1>t?N$W2RQf6OUpTFE29!(+N zqnQ@1aWj+*K?Y>2sSkB|-Y-%u&15;be$BV}>R4e3y?5=y9s;aAlt>{>bY;j#wyXKiSfEo^fpE#7{V|jDM?YLQtTlwpS^u6xn>?z zaBE^1?rckhPR{O%iJ27#9$s9&*)=W$_Ums|-#AFsA z_sB7lm)B)IlAcLSPJ1QdZ4um$=9)x8Zy3uT>^>b6j*#>R)^f`wq$4nwS-XwyKi3!^ zFyw(cRBAQ%#AgBXPDiu{Y z0&|Af1{NWyvgYWpqZ^&aS}O;~9!HZ7nBAgHLqv^w%oCZLxy7i4Z5i2T_)Z>e^Wk3> zj@j)caZ#HIxH$*t&gWquoa`NuuH*Dm^&@#et#kjhFrCrfFFai4!wr>zHW;2V7qQE? zn98&rYT;fq*4TtCT0Tycq2REj#BK4efQub4 z^DjKRn9w(2S5<9AU$7p@3x#kDY%I^UX)Pi45L+;A{V~+%Jx@db+WJeZ2#vjoxc*%8 z+wPF1Z!dn~&10w&qT0mUD{5D*<<-%i-bIFn8hKz89ffl+@P(>Ly>K;0isdi7R12nu zA>b5u5vl9xTJAaUfG?WZ9+50K4>^G3?)4~I`(!`tfVR0E0$y(%P3$d0onPDTiJPp8ym`N+2T81KN*{6CUPe5cZRy6 z&H6XHbCYVPMcO|u{1z(Zq@6;0niFM1cVHt7G|;F+Tjqf^m-<|RU`Co(A7jHS#u|LI zBumceoA*-c9m4W3{fSR51hWl`52R6(N!SLLq;r3{k}uddEob(9>=)ibt~x)xKkW@q z?=+QecB8$~Hpw--hlL2V9$$zyoFaH3esA0x$}1ZtQM%WJWqvf_&NWe2T?fwrYSLj; z8${D?xQh?3&-|<0&uV56Q->n#aHxBs+?!BIc3S1_Uw9Yaqa83E;Oljb3`d)B6h^;@ zYL`o)B!4{Ke~MzC#k3Rbr{-<~Jf610jH*{AeP<*d&U1=X{;Uf*u!Ph@TL;7)asOFR zG+x=DMgL_i$;M&$KzlaIl47TlNA^b+b6g!(n8aT0V6rI@4b{@*L8(ai&X=EbZC=Ol%_1TyjKwnHIJF7&i$2`j(`WEpCkwnN8O}4MsuN_@U9-#OI5W z>VAD3*PIvG3Nv-B^=vcl!%Kx3lxY=A^Rc!1uS6tHd4yYRT!ZkyEWGz}Qy*!=f!T@AzwLw}fP*_WPZK znCXZzcYS|B+=41#38|hmxQ?ujmV6pj*V4zaWHDyqr<8_^3&+p>`RXvB+L~hBsn`$Q zMcIP&f@wjjZuaLL{s~`V3OpXLmaLA*&w;(zCbPf%R!vxpLhfEtyVhq_ zBWvti=}?>5fZ-C2^e9f+tCAK~_pkAZf*3Qj=EJvm8=eI=7m^3&q=_W+m2wnPsK z%?uD8E44J}iF1m^GYg~4pXJM&r=n)`o^IW)G5Ki=%J8h{*Zw*FLvw1vck2{GDY#KT@Fzm>jj$`Rs594gct;PdK;xD zCz=fOIBnc+(EK0ly?0ns+qO3xb=yExlwNczJwQ;T*DVN`L;?vtw5^Dc00}Kr5eryo zN>{27LP$dBp%+DpfYc-)T|f{*5s{)1@QwSNz0bbq?EBvLyU+JN-*=zqtUp#}=9+7* zxn^5qjqw}5VZ~?d`cta=Y*10}fwiWiaB4m)0MlS3x|j`4Dk&FyTCenAc>8ro@u9j& zJ46oIVPqK1!>F!`I7`nu;RP@<1Ro_-VyxHGgE5Dzz5@C6C!IunGL!?L-4sMV;=7J<{XVo1x)P#spGG zWmYC}6w)xBQ8cMl7lSJ3HW7w=1jijw4+L+Z#ZKc>njP$p-Q zOf{UU)iSb|8z@E)b4MTpKyX;z93l17yi?rp`Z64yagKJ%vY9Mz?a=BWk@llc%>!1G7MhQ_pc;DQmBR$Bu^p=GC4fVJ`ySOm zo?f!pP2N3nU*0Y4H;utPxd(Z_f2;nNzk4bG_$9A7JI#WvbJ|adV~-0gy`xq9>db=8mJvB5*Hu-gUrfv%!NbLkm0D_8yiPxR`QC>UH4P|!(NOepS2$Y zWaQ=z8~P(vah`!@8t0R)H0Wr$oL01y z=cs#YD_Fkhz;A`47A4RZjSU&W{93mbIc91XVO&?b;BOz+`!hyvPFR2OJD>S%@v)9z z793t%Z-hc-5=1?ig+?6nL{A>2K(o3AR-dItdzW%afGaDCCZoC^aTKc&mCy^D zNjFRHoIHBtLb{b9#wO(F+$+~PzInLUrO>7=UiTU&$zytjbh$y!pn82pwWzlMU zAXP_5;bIRp)ajann(RZX9Z@p`q)vGEE*q6l0JMDYz*o zIO_B&Gdv6D&HG%gE$E%jflzg~0#xPcA1WIe3v)_J%N%8Pahl|Uh^l;goKQ}fu}zA3 zWyI=bw0%E8N~i`*;42$lRg6XZ5{29H#H9Mc;H%#O(th>9 zUHVV~H9)rx;Ff2oA-(&-RO%S~i+J6--%(8>~7$irVxe=KCp- z2dgUYGqvn)&i+*IaIGxWhk_`n?ZfGlD^&~naI)G&*n~I1IxUssb@sGI^>lrLY0)#0 zly!*J8EY%FriXhFpDBD2&P`;+IvZsp3V~C)-Stn85xKPa#Z2=YPg&R>x zS*|fM5%oNQeC}I;^%sNjqs@5=8&aYltaT;0o)BeczSfYH=H$!9$CsD9f>8(((t^T; zQ5kuou7e5o$T|E;AY9yGXz5M1mp`n^Z0`!g^AxtTbr>&k}ZOzoxcbeu7MRHB_J~Q@T zXx9EUow%7{gIVAZ%d{ga`!{+dBAMDX)|5cu0fO-e!36I6ys--^PIqiQS~`TJk3@^dolGEy$Q4{j@Vby^Wo(169ZMykWMQYgmo zQ2To=zPrt-jx}v3dwB05A{)@PMz*8y^8TxCT5x=@i}pi!aow`9xv8pJ9r%OBEf>T@ zHiD`ZzG4}n<+cn4MSjTg-i?Jc5}yw++=%~M@li<*Ozew(>kOQ?sE?_TMkXozj#Wohc5Uvg)z$mVsF686?!ZjtlLaIQ3erMNuZ^Y67(|FUzjcy5H%5Z9nwI z(U6r9`w)>~dn>b+B{u`*l^D!`@Y2Bx#uNEk?`x?ISAsHCymIUJRYtE0Xi8yi^dukJ zGi+DPvB#-$$w4|H;mICn|e@gHVu9a49y~;rfw&fp-shu@9@k zx%S?pRDwOagaF%02dd2FFBs!vq=zi?Qzz`eQ;E0#eDnWKdSArz?|?qyj{LUZ``XCU z>vG=#9HiU<7&t0K?Y>ydvl@v6*+13#bw_%F=GoYs2NKel-)jH=$0DMDJyn@k_)r3j zv?!J#HCX7&9o1iJL~3owi012GEZ_JZ_|=HyBedm8UQuc}eEa@NZas$$O>p|B=XS!R zfAP8atyO>TSJ$ma{zC?mw>~MSUd!Dn@G^Q4RfQIXd+#0pk>(D-i?`UEC9XHE-Z(!N zUTVR1{<1Isztb&$an7Eu(N^ETcaLd7<>XFO_(0RkC)@!!G(mvk$bE(b;6GI5|919% zjuYbPUsXpUO@wsBTA6RgM!K{pEbZz394fJc(F3Tu;+xUme^FyVl^sS@66a~`ExoNd z4yl-txo2C9mYMcX4RB~TvJ82i6RrCdji3lUP0<2 z#p0G_13_z>0+y^WWcWh+(}C*mfDY0i>(l0~U7b)NRcAaha7ZipkVoL)wl3K*#C13E z(8V^IF+I&nWb#mVtEQ1&gHKFZhTXcy;s~2vIn85eS~`SkbsjM%!r&wgy5)?NoQeOf zH`mh4igc8!(zz+QxZ&P1ADo%9*+b^!vnM zu4IgB$6c%I8Oh>Bpj;@>6gQv6%Q2(pgyxJ5qttykCRv83L};-5vVFh1sx(8cKPl%~ z4@9w^Ahh1a@*f&I$YVoyrJ_^SamD_q`5}H67~fcq90ZmNz6xD-5o$M3blTHoMJ+D; zlEoUei_I;qt&b%L%ah?r1T&QppO~2%g4EpF(|N-wm_q8wPjk`1uE^=E_9hTxe(C0FW(M5WF4R>vmy4ZKd6FCjW}yHh0K|mpWoVBaPj* zNjKZ@rdrg2j>;7*K(XnIsIn6L3&CbF^IjWX)v%B-Y7+WX{ftSw!8AD{09DPB|3 zSgHri1#LataNM_$Zd@I?jC;i>EHE^uO?-u@E;8~cs-x$#<%?4wNS{*_a+sbyJtx#j zM-!Ggk;(1eO~z6BA~mfFhHfw@p1t1zl|7Z;0k;cSJ?xN)93-2%@|YF%**}M;>n5=q zqRUqfdOBYcH7~ZuEtff!mV^ZXdoST}X1Gh~HJPts$d>7w8M%`jhja{YVw6UzRpqq3 z-s!1K<$-S&6c23{y}l-yOU1nn!d-Y(dR9(HGWCjDuU>y+;z~u_$$>$FyjDmL2Nngt zH;6lG%8m5L$UN;SlAIW8=8bZ^b@)_ltC)fU^YKR+0m~}D~k~PseOZR4>!nU4ZEYzx}+YmpV6=rLBHOSOd?H1FFC79dvwpUI54k(^ zpuhwcfom|dpb{PJKHFQ@KhV4RIgTXTS~NU)TDdm7pP&s*=MMG=q*eRD>em-_&sx6A zfshlEP8OtmH8|nbruapn>aPAztKD{ClAn|W98p6^9M_i|H>{uRY)2bc15T(C-;@jn z)r8syb!dq!9?Of9X{hQia$lWDpJy|>Jwyc+4ImJjx=s&ywxMJcf?Zk(6NEJ=qfbLV z)7Q1F!cqBl9yFSKEXb#>Y1VEa^Hh1kqQ-|CYExcazR+p^TI^nXTU+n$LM3^amiVlF zr>~D4nPp7&xdryi#!4F1v9)p#7m8g;VSt(8Wb+CFS8sxQ} z7UG)q%$3wPt%^sh5dGv3BoH#gQmPAk9WOcBoX~6=25Y0=lM$&^Y{OHX(1K~W9s6A` zTLWMo5#nDd+bVTPN+++rQPK-KznjcdG_W2abQ5OV%6TxcLG*|kemsO@g>FoU`}y0~ zxcklo`Nb}tW0y&8K8OjbsfQUJVVOTud$KyhP3%j4x@k~uy+s%fyZ-2LrcK8H#i!q8NoTeeAJUb7R2}yn5M8Wv zRg}UVl}sLmja#DYLT5^4>5EV6i80qVkKyGPDJ(I_M+|VtAGbKdmkHMNBnnTTvs+5N zd(ND#T~RgFBTD!BNTJvEVXV3dthC26W#3tC#m*=4qyfZ~k4*F%)n5_BZX&&wner_nGYQ zFQC^Q?t7R1nSuHbD(VW{kAq1)fSL{ozM?aymPMAGzS_f+(%X)y4rxrNFLJUPpJBxnVXJ5IfffL=DJs=YT>vkueF%+FD=0x%Q47Ku+oD^ z*YsvZk2%?eTI}{Ob}6C5M|3X|zGdiPH%>ht*y%Z~&W?-g{rBO2H_N|=!~fiAFf5DS z#N|IHdtobQ?*{jHgeVd!Go4sHM`Iz!aUzINSH3d3+0?XW(5-sPLRV;4M27Nq?X6sw zl>@HLK1sESO0q8v^LfhdL0A5HvPkRQSdPp7NwIt3SJWSwM4BuN$YNgTI_XzvpKLK`CzG>tSK= zaSNDPO&2T0ocMOqo2zw6|7pT$gg4DZ`?Bj7agilIF)5C@i@s|DtCNVSECAd#CmH_~ z`fT~iAS^FkO&$Kmj+*dlpA=Ua0$lDQlI`Wl_OR7Z` z8k_IoKe3sroNlteMe7n#PUW$2g6)~Hk!Eu6#1>s$^J15M-HW^C1BlxhQ9T|B=Y23{U#(TF}p*F(RAm3-p`ODeb5`f4pj(9|T z^&Nn0dL_BKwX^Tacfc?zeC@0{M^>V9(9~4tsf3`=-8=c=OBGFDd2b~yM?b=KIf)O* z6il`iQfI7V1xQAk7>czDzMn?-AG{?jGISgj0*dvcpw*KW-r92n9&3H9N6-0(vsKyq zp0P1yA*ub{%V`8e88y&F*dPr9(|cwTh-oNW3*H(ycF1ldES&Cdx;MuE>wKk6(y|v- zP-)_&%59@>=Arxm4(>U2+KZpfi|D`Q9VvtgDQ388hj78f#Rid|eF zbCIY{K8i)gbW(3Ji4c#4j7HkCSFJd98s+8+b;TWG;Ww3=WcvHqJQ9xg{$>^`=Q1`_>)OIMT_yrhrOyAGanM$;kql zOc9?~9+Z?1+qpe5kg@@gD(2XL1}3vr_Y`%xtpUitz=(wSRNW=;OTj-bI{BparotZH zX;t%`K6;dWk!b}?#uuO-jN(E0|NxU16G*P6S<#bu4#I0EO~OC$JW??R=61oga=>wN<}59p89FV zgGc)pI(8mjQbmE9(<{i8@ztcm(51TISwFm3m2437*z1i`GE8c5&`GO-AnCyh3RQl9 z>TW;M6RFwn>*N1!GcS*EJ;}z2@|=KEttLW*|8jayrwZ6k{CA&sxe#k zx7`^I7LBr+XJ%eTP+)%fpETOQ)lD+*Z@%()B_KEn77)qqpVdxBSF-mn$wEY2%7~(b zi*%xJWdhF6uG4Ia&;O>9n~ROblzZ6eg+P%2n#WZiGbBYD+~^W zZ^~iOm$#x)lO#lAtRiL&KvriC%g7eGkyp>mi;QR0j5b_Nj6?xME$9BC*Xh63lzICN}>Jls;%)HGN5E=eyovimT(h@S*a8X-L* zmTq&X9LTU_RDVLzfJ&_=6w`O=^7BuK z0ij`eNh}gEMn-0{b}Eb7tj0Fs@s5kleY?;z@EJjOY^hNz`wl?lxwqdWcPlk+Db=ri z;}XA>PU)*3pelK^S%apNfNLOv5QA7(mLh7mIzQ}X8fr>xQK`@zH_*zo4|X(&?nU|} zU7HAXQ+Cweek}U#6mQ9pDoBLTlBS}hIKS$cEX7?bn!^?6hnOGa8wmSW*f@M<)%7ku z?1fewk!Y-y+WX2CaUmR8+y5qi{iAvxY~a>^+dQcmEin6=KHyqkd0 z?+y|$UK_z4ug1%4hMMX(Y~2b9(hFkC56B#qoO0YC+57CzBR9Y@c@8wlKU0T6$aS^J zHo`=;w3Jh>v{tFURJ=DXLhrtY$hdodWLpjiRg7pAv!b5DT3OBbfx$%$iOnx^KCZ9G zGVwhG>x>0l=#1nHy~)Ck>cewC(bW})L{JS@)*_ST%v*+(sYWoRCQn*Q(09N|6hr0k zr?zb3yg;u3B{1J~R68W=fr&4tbt_1SR~aWG#PFvcx96Lx9~&9;X37mCV%E1>wG>;A z8Pa83Fhk1O%Kq~mRYE&iWxWn=oaJCweR}7@J^x}D(*^jsZyJ5GI`m& zPZ4z0+J|BF25mvkgr*vO^q!nhTAINQ-#VE6Nh(jOaJb8fp}KOaf?b96@CeMaXCxI% zGBP1nVpl;Wp#iTB@6@QUAKqC?&9KWuuKGxq^E}4gAGb34`NMZ3D2!9;$Zhr^VQ_! z8e7tS$yqUlsm9_X6}oDlW0wn6pdR0LWNfRd>uT1aURwTVS162EK{1wTZG2e{31FRf zZ&!2MTRk8))=V!-=g8dPN;ZhfMUtM8N~wb=s*IqDh))+BoP(RgO#-W!)D2Te zrl~cx=@>gqtDGVq%+bZ%pcg#PK0gQbi*iyFf7}nGHyGly6^$#2;XZ!yvBqmu8boaI zkyNTsk328f_&hWh4y}=MtOnfvA&UNQx4j3Cb4+on`a3Db-vJU5-c>t#;q0MZUYUpC z^CH>1T{hnV@$uF6yV?3X9^V0nn!uZ5fI~$$em4ZQV|QEiIab%#zug26gx_&lLHLYC zy_Nq}W_&cC0wgbTHK zGE#qgCd}~hxMJMHtpIKHLrol>Ru1usZ?vJ=i$;;NW~EHo;xn}1;~o`@Ac%1%lowcX zk9Cl{-3RkST6LHK`r6R6g0dZbRPS-baHyuq{ZA$b7{xwn&)H29`O0tVVUb|^4dl?t z3?Ih&!56g8yQ5$E*s;!U!zBWb(0jdDof&IbMb(g`=@dJ9m@#!pKi=xa#?XrXlWHVN zH&iE3A!KZOxQU(Uv!wro9nU%MjJKwCCJZN>Fl3dfFW~?2&f>RY@2$Lbao--l2OZS6 zK&CqmKSF>0tmRDN-+8ch8X?a)4mv0v913i=dBK}lf3N+~S+Qe|9bpgFv~L}m0;VvZ z&mI6jJ$ve9vxk5`r@QE&HvFZ^_x7zWFGle{1X=)qTShEb3hFrnck$!SGs{$lueLK?Aj)$)^s4ln!(z~~x9x8NRP@mLLLTCw#VeGHKpfAxg!`QS&+ zl66mXPlcZphX!v$4S%KaXU(D~WekYUpdpN2a(=;v8%!H$D!xPUhs0pyGlii4R zMs0%wF^UUD^|V7q#`1c)Csg$6nPHueI}QJSCCilgu%n~&2KD}VwwqF7@7t=So`98) zF&pik(em0J-0q`qNBb+QqA@B3FAd}>N1GkBDknd?=`MELj1_F>=?0U(C7(`;1Y@3y zH5IEm`{LK0Y|K$sEN>x1I{uya?>6{hpuvqZT&d&vAO*>UM;aR z5ej~bzf_s6D8+N|`2?e#r+C!eyi!*U*}}7{oXYaKLiAo3)lj?QOg(9QgqVa4A@u{k zL)tm;I}Cd#U@T6Cw3CsMm2UUZ5f*g;^H3P*(56J;no|r^5oDs});k0T$VNeT7`HcT;`2o`rrnZ83V!zGu+&_?{>-PSwD1Fe;;(g_ zxRC$6?Tx)&uus(J^|{;C9F4i%c}NttG-J={C-qUWO@5cr=KgEV{5I_K3woZ7Ll_PVOs6#&0ugf)jZaq+C|WRh=yYPM>gGwrH)>TMONNBzestjh*FxaX?H^5LTXo}?>Q}eV zXISFA9O;={A^GzRDoSW@4o*d6D;Euiypy20-rKOw}L?!{6mc(){pj+IFRh1o>jE$`v?a@{UloD9p z(~R00$uD;5oY0~oj|U7{tVgLV%r${3uEO%gkYHj~=C+oyHk0Nt4r1+Q5fCHRBZ>{* z0ia9`jUGxYSZ7XgYN<_tn}jtynC<9MTevYYR`;+kRy0i|2dh)wl0HV&jbrC>$d0z&jSsX$ z>c8@qe6WtystE2+fG_2j%C?h7lR@Kl90G|Cw)0HpvIhjS7M*povObH(iRP8*^2 z$q1O!O)ZE)&;$uaEKIM?BFbx`(yu=3M|cFBuhT|kEzKNz-P`iSXS()e@+{IkE7cbd z^|y4B0*1ShyL$ZkKP(l;xe6@0S!c-BmXi(*K>gAa;O$w^-V2$no7u0<;pK-N`l-)5 z=gx6rRL07^N=b+s9n{E1xs}!+X-v@h_cl#Q%PfaAId~2b5n-dG?RFs9NFjeqdgvO< zzI=Hs&sDe;!UIusRar~Z)B`qAc&rJ98z%Yo7NyB8(nx=QMBJ;)IRmQI8C3@g{t;XJY zc-TB?WRR(jlyooS%m#6Fz7f)4&(1b={RRaUw%4v}Wni*|b03k^0309;=GcvyT=UDo zh?u{jL-D@1luo}v+xK1ku-n9BX6CuV*|1U%m|DVY9%*p0(ziV^0&j6__y&4Qr0}CW zB&*mD#%*QIl}0orm^Ldmmi8>w6Gu6K?D2iQAX+AZYp|UTETE*-z(Dm1guI2M^KCR#-V`h*Tz3pkxk z?_8!!=1$z@Ie27Zu)2)lqD>5j4Nm*JkE@C?qhg$#7>TGVF~Sh)O&5Yma^Kcsp>0m4 z5RHbY&Nva6vgPy)3_Ha3!TBEk-3enEjI8K)fnX zg2c?XC7aYjlri9sJ(G3&XC}#F7No&?`o?20Q_0Vo4$!vqGV!_RMC@k&xXSgEm2dkW?y2eu0J| z_YWVSl8sD+;9$Zt)k?c?x339>Rk{0mT?-xP6cU_X9hbSR6`;auo-QX=W`ZqWeS^md zSf5Y&l$`azuo;^nd*v5o>uORh&V zGNjo?Ca15u$midLk}%^C5CQ_O-U4p|eKYu_6m1?RVKcW~{9wSToQY>k&9PJjIvD<>9ox@87em5jLMfE0z~ErW`h?p-*W)u z=uvAjQOinl!4zx>Gl%F3jxLOfw+_-MRTMue%C~%MHPfQhUeN?p)6Hr zrYh%AM@B7(&IdQ0U&Izz*3vhi(W0U?@X!i2S1rNp)^HCw;N^ZisWE~^h(@I(Z{FT$GO4;IP$M}T4{lmfvzr_XrQRX*z#%&PKyDCZOXe5N^{b5+Hlayxv zXOlCbMPy$OpJR(BHlr`EeC_H!>igN`@W!K1LrzZQ(T^GReIJxmyt8dnEA&d0+|B<_ ziZE@z=FY109M!p0z7?CFSqL0}o=s++&F2nDpMI8cnSj#wV{^S4^I5_0u*lp2#%Mpm$tMA zJ_>Z@x7;>8KrPVX*--Vz&DI_LC7`a~HMqzK*7Sp-uG`Qd+BwK_@cBH$z6qBU!hy|1 z)=9jXtknKg6`(cKCG~<^!Z<0wi@3kEO3WxM<#S3ScFx{-BvP1>DWbB_gMTwCl=psD z?}qsS;SqZVwI`DcL5ZlUmH|DMFaE4#iIUTU=hRO!0EzxGT8>4SD4OrX zsJ~p^t~*s8iZ&mHOvFgk-IDkz<)i+GPZw^>X&C28am6DdQp7k8Y3J&+Z9J*tG(k+$ zHZ>aC#gW6Wqb#DMV>M)imZHJ_j}yN58<8iX?nU8Ejp61fqpPg}&Rvv+Ps+DBJe3g? zgNlLy-1Ac5YL3nT{N>PSUu+AdEmqYDdNem`Y={%9>KESe*t{f5s=KPNX5KM5lBjG6QIl8q_VFy)AhAygZs~YRD2rUiO;vvM=RCEI{`YtXi3b!`1bzopLi@ zg#khYtT7y(EP?@QpYX9Ao!zblW$&3^*OkcbAojLtrduxsyVWG^iK8`$tQ>6+!HJ!k z{}>hOoYs477-?@h;9G^4G1;kqshi!^wR_~nA%PH60-?LLMUFH`MOux`C3<^FGMtoC zhcBg7rZ7ostkM2V-OzPXu}C@@)sDGuen5m%?@B+f`QDULo<1}5mtd}x6sDUn zqYAItrtKnkQ7tv$i|}o}BndfVM2ti3XMrKBslLbZK(V@q19x+P3nWHvrD@en@{n$5 z!Y?ARN*8lnkPYbM#R)5_DI%$ePwS%ECPAnb=T${l7%|5~BBfxF8W#C=#=KyqFGlYU zt(I2Y-$|*EXwoh#JTv~O}w})7fd>N z;m+}%ie6J%<&j+QWallByRh)yH*Wy1AOj<@niiZc&(u^bi}^Ldqu97kZULl7U}bkC$Mj z4|ta2VYVG`W+9EAmaLtOz$wZ2&U7cY{JJGuTvpizlvBF!&!K(n}ha;RNLq{yxo?KGEF46un{*n`KG}tsKLzIa&&>a>CX|>al?lSBP)5{9Q zD=3|j8b5^x*I5ftf9uPIclJImF`@wr0$od}!a-@{#(DS*-eq~y#vFT(=%;i$&uAV< zdWyP)1=Xs6R3(-X^O4ewUqPNq7wz;4o8G|vz)IlBgHX!#v=ExJnrvVX>u?|^l=zXI znaC`PRnb!O25@+#_1HxlZ){B(zl2nO7S7C)(f}e_;2mLw)|b`GRFE4cBFeASpS^*F zKMWsET~j!w$}E!VNwpap`g~*CPTt}>;Fj~7R+t>#v*a@|YpL6mk+9Mw!(aXOt$`wL z?~$+3Z`vD)>FPAEDXDkA2J{=W<#6Z#?X0vOw3GH-Qdub)T!A6cM~?`f;V6NPuk0Z-Rr&Vtm_pHy);k7a zX{r9#k}I5VizQZ3KFM>V$fXgH%RyiA1+SGA^G{#PY``7fTIw|l;2T+b$XsWklT9MGY)coO?*jt79kA=PW*t#d zNS!T1`r`m1lRE^KMb!-{Tt}y|^`NyL*m@%0bHKfBg^D2|de9ETU!Y>s=5&MHvr8Dz^JpSrW;@;*7@R|b; za0MKS_VXkQz$?sGrQa7B(5g~^g+BMd_7`K)Lt&-$NvPmGuT3sOUlbSY9<-+g^H5$N zHFXaQ6}G-#x&_NdugOOol&w%9WQbC=I?SWyJ-zFr*7wHxtBCzlZtVPeUOR2vdNkd% z@X#qkaw~aY_qqlHC4Qff+ldo}->8|l*o38YWhRYYPZKAf!4;ccrD>jEbMF=U$P-(c7_GXx`;$>j15;!eW^m~ zzv&P^8vLzr!hPft38t8PCGHj27Jt&+u$0i1AAhd-Hs>v||6jjEIJtk?{#dxt#Wm8` z(G;l&^a`u`{llWP8=Vb2?|~`7tFn1mEuj@9{eIGp(l)=fXW4Z38v2JGeE+E|z`2~S z3+ogF?p9)TrIqBY6vfd0mmA- z)*#!L)fsO&J+XlCfgYoTYi{U8&Qk-8iHe5svH==U&`PX$sm4Ztj_2uy38V5$=w_8S z@Ap}>-eGN@Sr2@*Z4vBWK{b1bq4KvV7W7PB_Va$j(N~v^mmNoNd}Mr^dkzPV?aQ$Z zJB{W8cCw23%%?To*B9yem9YF_&j)tvM9HL#(i**|?eGe|Pko_+!aGO0E>1OAvqwI{ zDY!i26E%TqM`IhBz2vm`FKkv;O{dsc`h5o&JM6=@*505NI66$$2Am)s^vEBTkA8;j z9ew-0v9tZY{i>)*hnem{>S|NJqB!C{CCq8ds+z7?JdurA-%<7}Mu4SRu%wcOHn~FL zkfxLP4G(>jk%LFP^@R|FtTES|=g?2sbYf%t6JX&1V4_0)%v3z5mbXJN#B~oZ zx3-(Exh8NbFyv^wE`;AvB@-o!NNFqY4=-SE^H7KSVUdf+;KM%RZ@#1yWze`oQ6o|i z=LJ7;8QtNr0WTdFp--Y0&BPDdybx0~c?41gvSUA&45-xE1P1$v-0WC3l@cJD2&Q_5hVKL?wJu`0y$v?IulhG%AJaU{6r2Cmxx$zzU&eiq|^ZGxiO7-iofE zLg51|3OLrs2RLo+x=ulyVkt6-%C?MH_)sM3Xt6joccp|4n(~6=-z^Nv{SLtLT97MR z5iey#l15{g4$$1OlG5zc3YEVNotx*^9r3K%KpulFr<#hvpu3r!v>Z_@EpHOU_wi`T zQb7ZyEX;mQb!&T1u2s&fty<}(K<6tAEE%;yv7*nhEJ#14C&^vxv@q3sgj>!QQ@fkx zehf`oKabV*h@2m2U{XD4(NTi~aro;64v1te3>qSuhOv?jebeUkFhI5Oj@hvDnR&jQn?T3Ez zfD!1J0{C7go<9y+Q4<+SyJuG8?yu9L`S62v?&Kq9TmAE3%)O^kA5C-@56(G?U;G*+ zx-;*`n-lV&rENoIPSzjdjy1lU^?Y;%Z1c>qm8WuD9S>W{f-a0zw(`^l15I*tUqAT^ z%7lNYgdD+&*GpRt6-kL52JKhl=l$yOy4@-5z2$D|?%{N9(3#!HAD+oaO8`d4dX)&r zcBDRGcegk?xy36xJbGemuXmuI2O8s5>x!)%$81@GILy3*IbN^QhG-UQlr# zgZY0jz}sR@-ZPb%uG4?k8}=8EE`E7wXrK}gI!g(4hLxoxTfanM!Nl53p^>7}2xMq3 zeX&O814omgpJ!zJ5uTFz{Owz$BZ|Kw(wJXH(^wUW$sHUAk0XqBvqZ9s6j{cFGxFMnDnv2=1n_36VFJ6Ukb^8^==13J{o@&c z2klz3FDF$$Fl$6ib!&}UF~r2jx6zjAiDruQh^!He?PA>5@Hd02CITa?WAe=(4JsHJ zWMdkzhXxHR@o-8B!rCclaaY5GSM=;7f-RS74)f|~wVf^BO7giDER=XlDZ z-T7qNfAR>Sq3bln%A_^P;(`A>i(Q$vJ~|Fj1^NNouV2f{op@VsB41@0to}#S|7nt3G*{C(*=iN!wDzeE zBFGNgsHb8TD3v+L^u?H>QdAxz)e@`N7@y->Sj%~rELhhnEV&YG2B!07e2mSMp4oit z41Q(NW38VRtn|lB{c|4&__5)|kToW%R+o^H=t>-Wi?lV!n-zg@^nC~nb{S$iwXt@I zM{wc5A|&-7Br}#P+nV5H83-;4RKur_&&JH;bpO?)=if&f*i8Vff7?A-8yR`H>p^B9 z1cc?UDNoA!sDN3OYjy?;ZeQ@q9VjoWDp>Dg5A`RTC~as~)cD?O$ChWPPII8He{|FG9j1~lbV&Iiy7M|RyBLXX2uO4taFR2 zo8gUMy|!_su==&pG-B7=p|YXylSUiicDuXP5 zifzh+n9dw;nYSWoLJF^shNGs8z8GBvHM(V`>;F}A0uPjk{?&mx_Qoe$%^c^tZd^r4LsM&7Cu|E1#Zw_ehG>^@~$xF2xuHy}H*@2LihSm_} zh_j>)GYpb73YM(h=)E4+=JP&lqy->0@+T)N(ZJovA8^x-C!qZ`!W{q7`2RN(z^A{t z+9;-Vf&l1`?wXsbI0eZ`7edlWMgP=zPd|7`Y9g>uD*I&KtdgWAw1XSs*LX)s+M!Ug z{^%WqlTom{9n9DI#Q83>)I_aYh&8!Rr^~h<&bUZ|i5?O-PIy8q6@`4VT1*GG!-eb5 zqm~WYw9bvhKg~6|{QMsh(ce?twihGzzu^3lyK}e0650YJeYtas8zji)*C%`j++BWM zHVuvc4G}~9mvjGn862vY;@_Egt_b`gSM9+Z;Jz*J?$_%7kcd*hEqCWm1JCON2M(;O z|12aV`ef|(ufJjpefQkCQ}ZL63u_p%w^~TF?@#%B6Ss>Fa1j%p+uynSZhy592=KRr ztP}DD8HkL&&8a#3Hg>=IS>0CCf=3Is9Ns&bggg;JE^Lid*V3ItS&VTbZRFm$kY>59 zeb^-BSewE#b5Xj+r`&leA!+202+41W1xyP^K+j*c!QR2+SNC`W_PClvIJuC^1rF*` zSfG1hrjFuam{;*KPGy-6Q(`nzP=#y1GQQ}3oj~I!2>X4os@~8_)2eT2X;JDU*HHO5 z%Wpe_vY~GQQ2>|s^v&>d%KJ`BXP1ACGu3zdyzl4pYp8GfPyc=3|MMn!#u)>19CpdK zKZvG;j|BYgs`+D-{k9z%;#V_~yl-4XtX1H-S&WQ#hAoV>Y8%fcbRePojh)RF#~Pgd zR}9uQr5&i(;qvs_tqT@)oj+5+!VC7`<2NY5RV}Rx^RGB`?r&dErC+2K?@BptwM27k z3bG|{_Y{qOQV67;=eD^!Y*+cBLv->{mb#teYo|w&ZbYo)?Z&{Tu8rxR&NC{!o{s0; z?w0?^alf>Wf86)pTEOcv9_3{pA_Z#EM#Bz0trNET<+mDh+xd6^qNCy79P9GqQu70~ z-?T$SAZ01A6q@4v9c{_UU(ech+NYSl>dH6C@hDBS4B7L%k*F>clh&C;{*Vu=B-CI3 zCX|_>bNjs;SJ#`D>UP1INrk^|ANowcHhC>BphwkV&=;rWn{c-SqQ?8+j#j2Bf8V{c z9BuqxZGTtva02(HbWiOOX^@3}HsX~qB@s}DcaoIyJ*mv4OWod;2FGMtQNE zpmd~;Iu41{8bX=7P0?1BYKR#*Xc(=2BUR}gsv;03$(o_Ch~!FGja(E|YCHS>190?u z)OXX4ykEJQRoUh?nNx4x3@XMtFArf=ANJ!pT_l|Br^d&P1p$=W7Q zQ`*|Amc}9-9Q|GCMpgu1Fq0bUP%QMD%H6kRH8?A8zs{x4d_E7R`2u-#>i{yxEW6|c zRGx!(e@zI24fHmzH?p*%$1^UxtCA?>uT^bfkn6gXB{Z64hhZH#>wdD zl#_0{6Ro2FQWKw^c{YAOYwmZCY0qgLg8B-*Cb$p1T%4E8N6pO#7mz8k249^GA-56x z$uG2(m%jG0=bUq_N18m;K%<=hHZjVjn0BSqY z5XroeaXWv-OmzR-RR`L{!@*)a4$_&E^16$AdvmiosT{oUm9|o)Mf58(KXdVx^;00GJ4}-DO=vEv@_+5`w z-tm2F8aa4!Q^(|34`9Wmjk~Rqla-3@$ekWtNipM6Jj?d)KE(ej&Ant}mc!pKxU#On z6-E3ysVWLuKxKnlg8;mCssh}+UD-fF>=HxGo2^?QOm3xk4-Up&U@z6vq>2^zV9Z@+ znG^7r1F~hI-b0(D41nKWAI@J*j+g{bL}vXM2PcNhupjJM8|T-~jcKmLusM}P695>{ z>8HENnk`o+0!G$0+8OK6!0xOZ#8YS576oP8mPz)(%a4>23!d{9H&&`U*|qR1DL#hj zl{~D*@fE3!VLtp|UU)QZcf(W2M|3xgnq#Slk_SZ7P6C%FTm*;#`CJfcQqMz&h8OKk z)jONVN4g0LkLweQXrZEJ;jug?lT?;T^Bj)Hxy}MV8ZdSj{a8UO*&ac8wGAfyNf>rEEfyKFvU9*ckX%` zeq1P&pwhL3%~b0`W*1)eTmJmd+Dkfl{=DghAx#<6;?&G-#>o%iu}-TP67dP#<^fP2md|g_)4U|3 zR<~g$Q!3A_ego?4;K&V^xncb{KgYVf_(k#`pvPX1zUZ01)IBuTXfk@b`@Oya5ha{p zeQ4mq8?Z+0fS?6Ia%m8j&Hkk`+(F}>X4#`&TK}@k&S-Z~Xo{aZYE0h?YepOkxwR!+ z)U(W28-<+SN@Y2B!FpG%_qt zA0i(g`ljWmZdd}*{N*`VlsB_=-rr32=vo`SqoUK=B4&A$WMw5cW;fYwE5WH;^J9O* z4i!jyBC=(J4yH@H_Puo>bAa))2I2^L z^}Y&xmWQoj*3m<2Juoa-LrNCE$hav~!q$kGxTI9048^hOBhTU-cG029k+mD1M-#GX z$mlP*7Cml7Bw-hx^ZXrPESpSQ_qTnP%E2U7{TuQk&d%}^hUV}_xl?vyer z`_f)KAg|l(4)JJJr#R;Z(JvxN;X%Tj%0c{nXeENfNr;9IEREU|CCO7ERK2%6U6-1e zBpZ!I4b`D-RQ4y4^Y`miBmsxCDZ%nJH@cUwD8`i%8F5H4zNngr+g*J>rHe4`*XOTv z!)mqU%olw_dl7V|O>*71YBE9JMk{G?D5b+DmCz`!4IB!T`i&(A^N?Q|&6}axhnn^- zJyu4Ut&p6AT0g2_6QdzH( zPDFr;N4C4pM*zOS5fqutdrZ{Ym;UfA$~%MJZSV`&5MNVUjK|vKY1h;V8=?%n7Wu)t zzHj4m<8%b+3e8cP3Z9y8H>?y?`o&7M2DJfqPOaN9`1zqs<5hn%x$_n}dL;&)0oKL? zCeM!Jca{!m`;fDW95n-?B{=JcwMgp%dzY+i+1)e^Ni<1a8=;n)&NW<(g3rV#emRtw zsT!4k=H`*1p4;JNnzC)ld(y?tey^ z*+w9YkcOy@>b=dMaN-_Tuv>(6e8mfWC@&(=>S>(K)}ZdPy(D)vjV@bDTxBMZ4*CtM zOT>&2kQ^8fG{|9Q)XlyL>+7JwXwe?Oo7q4Y<>W<>*cd=+I^qTdtmQlX8ueE}zsY9a z*4lG&VR6Iu5Fx7obBT$0ZHN$limmg9w?%iMxTLV54lYl-6|r9Z-SUe7Ltg68u@5Kq zP4=#ZtHSdrLGy`JPXUn#z2S8yV<_NXpF89m9egJdHNA<NIbsxK`Z%!bre0>6~%@7<- z@cQn&=%*!pTE{^ai$^RexRJ?ax0*#IxB?fer1ic0zPH&GF5Al)AJ(t>&;?!_?B6JN zneS99O-S>D`Iv{MJj3PH!dSIb7|ixj*YF~R=d##`S4W=|X4~R941M_dF!BQxW#8UN zrem!FJfV|4`;#^P_cy>OJq^9pX!HyJ3-~8w^rz#)e|FGaoY@c?6#nrw+BuPPdHlX> z7Pu@eEo}gQ;qXyfC7LJcsjJxXzQ@Us4uBrA;Qzf(E1?d()!ln1hKn))PRCpb5bpP(q;kzYzGZLl>`nbkgg z()%4_A+v*qjxEybl&Ky;aDCIWXWOH$rqh>C2me#H?hOBSc55lb7iilM! zwZdNQOJ2~aerE-d83XJ)yh-PniHwDVzHl1vntg{9d@LE1emTG@u`kV{a9vk#AhD@+~mzZhbKh0JO&~MBU&FI92P5|~Nu%Z9J^TvbGn)Nh6#{=|o&rGTSQCk2{ ziNKc}Hta4%Prdh5$<5w%vi^E?$D&ZC<|yI9BF7!^b)%BMM*7XL}FsKl^8 zZT<={!(PR(l!QG+a~WRH)f z5F(sQrd-eVB<#GA$q1L=e6BGY=-=v{Sn4o{;1nY!ULk=%hK!yr+Qyn|x!EzvHQY8a z{R+&_^UU07-{9<6_Pv8XFPQg5W`d&K`*q=Q=+xlcEs>h{rS~-|1ELNGccZ{6XlIm; zL5N%)F%@f~`dwx%wMY{2QC<>mV)3RtJIqVVLSr^6s1+AAACuV_onqfJelm;ps&{ACRzFeGXqSK_KkbAqMzf3AvJz>fK zui$CWiw_iQ?CF;-LW55I3*fciMZ6aIzOR2QO(t|yO;7UTnsv*4*tw8MjpVn}8-AWB z=w&T500Re#tb!R62r}kuhr$CT_-4uMz$|{#v!fGLGNnhD+l;U@sUWLeJFrV zAZ?*MPU+JKO)m>SmF1-Z!E$QwdbI~S5 z!mojoi;}xGJi`-XgYURAT!D#n?7+MF?2pJSK2i^DXyaHsQ)Kn<3z|&mz3XvR6kI2v zY~#1;6V4ZvM{Lo~C|YVzvBUJM#0AKX*w;0tQmCS0L$lRk6p^^B3W)33aCXg@SA83y z8sA?FNL=n8GmE5CT^(MAPlF)WtKZ*i$-z%+fBEb+W$l0g2dSv_aFH9`e3Z$)LJWRY z1mlh{v#aBddg27#6g9A*u<3eNkHONtEO`)>E>B})6K~V~dRqwgZIk7?QYT$eaT&6R zrL|g-a;5r3Jur#Ro^om8AA&BP~n6y4W>tin1jLK2%x^O*wgS%Q2Lh>BH z@G`Q1`}oNR4047_R}UsYhEHVqe7+n&fF>j3J7VUmf8!_3Z} zPbPrRa`>WRhLlFOCqFolfRpAri!m^n3beLN^#ugTiUoC?*FD z-J;FKR1+fHjdP}Zn%>AfMapi%KOKd|akVu;Y~+lj=e%f&Fn!K`EL+j?m(VTv2fz#c zYrpo-81xUCx7d}9`@^Ri6LKOM*;O1aw+h)`PApkriO-bX6VRQEVygkYG$7}EPrwTS zl}&3IP{}ZhU$ibvzfGHwIY@4=&5NP|D8fss_$u{pYpUU_q3(egZHcY1$}XFXlZ7j8 zlbPlrEe@qeBGw#%@YM-{A}k>fRW!rTx7&CU7`=HZ2PF!7jyBWiKNu)25$vmrYR+rK zOo6E3=rVq5S_{XyM6jV%V_9!`NKKnKLdB3VkME_>-ONPLQhH&XPYbC6{SU2XK7^2 zrd$>*d7VuIztc|Y_ZL{HI=Lt~3jSziX5v)aa26i+^<62h%2ADeht(}b$@zmLF=nJf zXmVvl{=q|KMeER;W0;r>AJPWd2!EsfAr%e zt(7V`^gv}ur2hKqNF{?Fmfa}Dqvrrm&})ij^zHGS;yGZaoX~VO;{(GATyT!y_(W51 z6w~$z?d7)LWS#|Q`cofEXg&4+g%5a-Qi+=I?{UX5pS#eQV_ljYzPg`a;=HJ^t%S=V zMSZ2STR|!YYEl~b!H9x5$lLCb(b#?$)A;z)kN6K3~9oX zn=$3H`BOG?btyO8GU&{ghY}Q1v`w7sG$IA65PY377Pi!a7iL98v)h6e?Jn4mo#Fe_ z!vDJ`iY`6~x&t)@7?_){>i^cVQzPm}N(WT=2a7+G=dcl}6{!>EkJcUazxpDghEY+e74WlLY{nS^blrOR z@W6<)V#m)#sJ9ClVK+$0VfUbI3gLta$zh{@5)Q1$bHgBeN2>N6%MmM9rjXoPPZn`^ z#yWE4dGo|+eZzZBAG>2)-7yHUxlQT@b>Ba86yO|rp=46e&&hzno$3FeBr)NeYhG%p zZ8g?BA0r5m9bJsgNqmJ6x_MTxRi`3J{u3=fd^o(~mh>lnK<*2 z_Vj9r8C>jTS;9MER+mS7G~PFgSZ$~_U*a2>#+V*g*M^)gqc;=b*CD*R;Ial2Fud?o zJ^CB_xkmW?^}<*k%%97MDxzw|w*5Pb-{6)r=&#%K9MgKIn ze|FYb^nD+IAdFkom>}K={aP=R6-WNn01IRC5my!2ZPUNlF}m8!Llrj@YN(Puj4b_q z-0Wlw&C5vZvo0&jX8lHPuKE4H<`=i5r~kVR`M;FAzT%^i#;=#V19jxQ5A*vhvHL%X ztr&piqfh?i z29RTE2_!zeb*l0Imj*#ChP*=m-FEien~d^`go995*Fno3|LP#(83>{)wclw{33cdq zuT!S>sq|)q*MoOH!JYBOH-T1p%UNU8Bc)aCF8&l6u>{XFPqK{c@&dv=vhHl&rW}dOLCt`qIHzP=prK^Gs^UQ0~>_W7M zKK6NYzTLo&S0tqq(dG_=d5@GQi^9tgfXTs80JQ6K{9cCk-pmCP{=Cj}A0b1l;}GAJ z4cL?#(DeCBk`@wr)7T3=0(*)$#f4kUR8~i0UqjRSE&P=s`a}9pQ;C}b^g|nYB(3iB zsQPM6*`2pGynf2_ta`sbre*Z~in4J(_bxKd@?D?5O_1~Xx>xcfJ2$t|*Fh@dbHjiC z!T;$ev?@{s{AH1^7+%2DfaU)y8*)rjduqmPkRmfkH97V9` z4hrSLIQ}MUE~s_fmHlZ%j6Xu2n!B+?o69Ct5B*YEU4ZU!2H)QkECjR(Yq0Rv3xR!0 zclMRBaR1P{W9`+W#3iFWg}~qwjn$*NBTk>pNA37w@Ye~JXwAXYhZ%>6A>a{>L%M^e ziW9#&gS#eiiog~r+fDnkpFiL|3`a{VE(As?%?EdIrGF{J59w9>EbLu%T<%R-iFJ}7 zI3DO0b$m6Pxw=a#NIWj=&6m`^5x6H;xM7hw(8 zyHcatNDL`%(ZW>o=a=VT?Khh$CXO=wf%KR2hRWSd3IytYitS*JS}c;n1<#kiK7j@M zVt1~toNO5`7?IVbSxgEXx5?I3Lw+WZiNOBIxhkGUUk|>lF)aMf99&-bC;Q_|Hot*i zGlgBAe}vwXC;m&>{~u=7%oHBb!d!i^zgn+Tng7dMc}ixpXm}7_cee+qeUR<{olHaM z7bDop?BQqYaJ}`}sTp1Jf82Th_W;XFktNyF6U_tGXV3l@zWVFRz*3Svn zsp-Li_3I>;JI683aHd3kD?~oeBwvvPeDf<>QXy0^M}MDfgO-p{9O+q>&9%(>G+olQ ze%sR|^CxHBrOhs({hEx}f#A8V;7Xg2?koFtPde0DNC=r*k#mv(I-|;)m2a!m+4MgE zEJ5CX4Pa#dbS_Sl4DRYepCl6zrWQ)u;w z)Gcw&>!!w3g!rx}xw3kVVhj)d?yQ*V8vr1i(;?vRgujLuFa_m<)2zS7hCc(w|EClF zhL+KL<+PCX9xoCXNHAB=t7eDd%@-4>RuKL14Fcjzo(VeveHCw~_%80`XB`%?Zw2ut zY*%OVD(`kYrcFF`1Xt~vpU>;;stsA2t6xF_t6`NH(dNYh5!+aXsU9I({J3g2rMaTr zT#c2K8OP7#2dnqCcBo5U^cWHsw?quLcPP-X>;;o$h8w(e9(OO`++l8KSYt-`!! z&OuV~@nd2V17$~|7Y8d5 zd_yKC8B-ubLe$`l3&Q^ZTAe+$_I$TIyg)EThR7EX4k#6)g=kY&DQ$k$68 z_;l3XsbG*CGx@+pn!7?(W-rgBbw*hMnwkitY=>&8;`oo$YSWS0Iu|}keQHv{BooBR zAHtvFz)w4<4AGiw6wl*wsC{A`D-8YU?(jx_;9QgMtcPD5Ly^}-2yJ=uxg&C$tH>nU=x{kdPOZI|QMS}_@NsMnCUPF%Z~nkjf4A-x!akVdlYYNo z5&72|@oA&owmGi=`r(%gNMzYG3u05&9Lhm|b*BFW&}#(^R&vz5$GoP{3bd)6)N6Y+ zTkYH=DKLSdD5X#emyGQF^J|5?BVA}FZKTaRIpIp``fPW1Iu^7R&Cm7Ay(S~FyImb zKPi}_tS|H6_`rH*qTS_zDUDBN4Y`}pWcbEu_{Lx1@26qziZh@7{5b&L`-u8=$%7M4>il%fMz z4~5(#={-qi%HZZC5O~xH0oVWX`oJN1?c9Ayy}HXxNQhtBwam8mQiHoelvj^$8_w{q z9BUmGg>MS1SJ`r_~oE`1ri9$8xoDz{j~RE|qFt_ZZq;teL&# z?~-E}MjXILPFYhF#l)(_1R5?Fh+%T7Qd(@7rwx0v)9Z~<1TCQywf&oYS((Eh-rpG) zE$;E&7oXgU&rK%W0E%W<-dobh+Tk|FUIV)$Mod%YNlhsqZt-O{ zaQ|v4XvGikyipD}sl>e&=l%~qZ4KtaxsHOwcvq?!b#acIjF(q;*B7S3l%4N~^?I{NfVUFGc?9jDfB6- zd~lbrFIYuk4Rmqo*_1Pp@WSldeXokCWIcgi;K83p3!DINK7KsYDA4DxE4zii{3rQu z(=K3*th^wLy=!TQ*IZPt%`HwZ!|%o3gq9*yHlrO%GmLA);9f;TnK9Ln)!zMR^wPk^`si^1XX4^JJ+jS zENqVs77Ef`cNTQK`)0!$Iv1Gz2-xl|i~LYSRk=ff-J1){Tx}0fmRY*A#Vmoc4eMps?4TUZ%3bZlmWX>vrhAPHEe zzQuiS$N(fYtsquIH@EEBrAh%kZ-GCKHoyWrhh%=iR`vtP?7@Ny76S6^U>m}1$!a9% z#;-&uoZ+&FRh9dS+q$|kbvbMAg?!zOl^=dV53hcgEo}D09BW4mY;GUF!+x9pN$Kf= zvQBZ_>}i~z&Tl!^yVK0UzeZhHGB0^ zm~!~ae-#*yOtiG5dFV|=R>BGdR5U-|q5M=uK?bGyCm?oum-Y@oqUKSa zeEMvpham+S12F3Zjs|m%tZw1?DjmM1Yaj&&0ZRv=G8d_CMPnO57>mZ}XR2^uSlfy> zGlQ%~rYUGJ#IISUr9#)vUDF%TX-=}lROHW1>S#v z%L-KSn=0(du-nrS)&adah05HA{xH5DH^qu`j~Ol#BL@$(K}w-8E1eM)yU~Y2mS5Eu zctHXe*D^)6?jF_xD`1Z6R9^;2zu*dHfJhg4K{T+CUH19Z)SgZj!XYDi2Jo?rmScQL z&trYTm&t%>=+$R3$B7VWV0B)IS}Tk#CVLPSB~v|(&`y566#41T`vX2=j(TRwlyFzT z`1reEIZZ>1?c{CkUvSN zI5YV29@Ds9v-6p%(3upKUcqt~{u1Xi6&&Ht91jX+sye-+KABa!3Xe*?^4_X0vm2!x zYu%p6wqteW_`j4zP$sBIGS0oMD9$UXylzTH2GZoqJHy;^X^uU?Tx~)^uYrc0$L9K4 zigaqpJ-bo~jlY&fCBxx)Il?Uc(F`9lLH|xUeI~L^>X|Kj%}0h+16?y?4x&{?_2YpR z4SHXhBb*9OHU=w6AAKn89$aml=?dzqd zs1zo-HW>3!OZqA5NLV@+%sCR`fDoSu=RB;8_%$={mJ!z2WRQ_0Fa=J{@aBUJh`UVmP$}&2#rUU`%M{L3X8cwx;JXtY z7*{GtIBxFx=7^E`!;`IchapL8$SiNo3c|hsm+x>yIUT5j42T4-Mm)QCpwo7?CUx9-WT6t{Uw3Tt~4nWh5Ia4r96*ts?Wk>}6?_yG zc(8ed?KrS`ANa_xB~{MaxUXiYB*h>7tr0gbqaNI3+uEf2g=Du=a#R(7F`uB0O%1Zo zI9rAJg)q*k=H9I1_Kq!dkkwifS#!bamB%>17iRA#Z`9K9F&(uRFw9?jl%YWrK2y?d z)yQ6AoeX2tGrS;DBhQ;?kf>6bZ-ra|=VevBwW)95lD_L&5iARsIF?_z>zSB*b$;Px z=c_lDan~fk8P_DJoG!AoIw(*7_mluEVtIe9AOF+;2gsmg3(x9#(h~Q;K@|71$ZpYP z*KFJM+;agxA+O8A*>CnPe|?I56r}38w6*=HZ+0%i`knvHZPDy(na-loL-`l67OH?d z%>q~QPv3?@bXJ9qj9j$RHviIU2hCNv-buOhc2SCLu0WJ|7d96XEk~M=yD}NRGraCc z(?NGybS5EkXBgp{Q(oI(#|D9c}sbiyW*%u7!vQyUWKFcdFJhMA)!+J2YEx=gCHd+#kO3* zlNTT6+B#U;*jzJZYV+cDUDBd)Ak7Bm`7EQ^gmBW%YnXy$p7l9pz#YiXjuu*g$T*`M zn%0@_=0zK~Er3Q&Q}VSrfyCfu|6xk{pT1kZN%C`gRr>r=!>kh%N^tXq=%DW=nHNeH zqHjj})|j|PKL42xMsFeQ2NFi=lFD0n&40P$#z+xIjtll1uIqDS<1rT!S;uF;e_<+b ztSjxrm`qstxUO?+ag6!a)y9yUT3L+Q1eKeLPFaJd)JImOc8m6hx`x}{ulN5ZbARzQ zk(F&cz;(fFuV;2F_->|ao(01g{$%Fo$J>QtWd}oDKjT-^+DcEXPtUC(wHfc|$B;m> z|KZ#D5Xl`yOH+II^lf`+|Pt<)(Pe+W=1XkC^QH`TIk3amF156RYF#K8&%%VpOu9mlK}~eEQcG$z*B5Z&@sP*;X!eQx$o; z{8G4|fU(v*>#M(2OJ_Q&uM5n@YAp?#O?u3ahbU;NHq*ZI;E3(^UlBZrgiF5fp1tlR z_hCGW#|(5g#zP2SUN}jC91q>~Z2NCHtOtqVtL!Jz(5K$)glRBM)NEqxpKQ$)?P{Vn| zJ!eog@2YGc8+GVIM}P9-Q5)O?>_)?`|5GJuD(f_5uIXp>qmdfLK|FqW7gKacBTeTQ zAP{rQX8`fJahJK3{sRmArMxnK{;ofPFCQb!Y!;}@@|;y9A%2Jj(L17l1Rs5TSqJd7 zM-7oI5uSdZ1qKmYu@vEqxpoG`!_E+ zOCfpEQHIMUie~};2(dSsW8(H8a|AU*=7Ux7e$%B zUHzH8Lip}E%B>SP^@e2SQ-7=kp8}8)bT*i z2f*YAzTvP_zKqWvx|o6L!?L60({f4PTuK)9L zN&};8=H^lwO5zCIgP(G#G7Xb=_~j~YS_?|fS*1n4pnk8IBpL?s;|6{HLHFPXlck}j zn9_lXCFro{twy|w|Th26_qG)ya{7E7&bmMv4)8Ze@yF7DL?2x(MwB~po zl>wgN9Guu$X`frxK5@Jnor;vvP19#p{kYSUZfWk)y^2Q0Xt!KZxc3uo81K}`|zr#hP#PCsx)V><5dS=*ug(0v>*tREZ7zy#!9Gw)s%6 zkum=hU_1GLcF^1&xH`o+?5|ZlDBinl%rz5-W=W=45Rv(Y2``yHuXv)GKKP})EM8rF z)y(w)$_x3C*KIV0qA_Pof2e(#P5jQ!$(bm>gC{XG&@I2wuYrkVZTHttlQ1@l7tx3cb4G=_m5agaF?n|ZMtT4P=hnPu%%sq7GQx|TfOn)Tzsbg~otAQP zrj*x&^~k9eYs~*pJD~d`uK20#WNHp5bT!*^{w9y0V{)-3qe6w-MNc4EyLehn#lF(< zG~c;VUd{mTtDzhC^;7wM&0Q#cmwRth=H`H)S)J93XhQ!4=(YOQZNlh#_1vP%{7!oA z2q?63#PNZ3Tn88Kd}k$(gx)8~laPzQNVX`YyK7-GS7;nmWJK!KDz3i{7twowA^&6g z@$cU2KYKT&uII~3H9xVB5ekH-Pqh_J1$?g8Atl=af*g|}JTI&b%K6qk^ny|1k~Ktl zEHji5hubfCSb`L4CRih{-n?bYi<$f!tDb2*QY;e3RAQ}Ga&6wSl_4QnU?9ZVjlX90c>v?F~Z{-(Xww1d8N)A9C z&|!p51L}?cClhNDI+(`6*R9heMbF5uXB{&D=&a{yd$>Au9=$8pA$zbZ6L7Ql+gf}* zy#g&)I}p3Id6o{y0%9HV 0: - # Read from cross-pod counter (Redis-first) if available - from litellm.proxy.proxy_server import get_current_spend - - team_member_spend = valid_token.team_member_spend - if valid_token.user_id is not None and valid_token.team_id is not None: - team_member_spend = await get_current_spend( - counter_key=f"spend:team_member:{valid_token.user_id}:{valid_token.team_id}", - fallback_spend=team_member_spend, - max_budget=team_member_budget, - ) - if team_member_spend >= team_member_budget: - # common_checks sends this alert on requests that get past here, so only the - # request rejected here sends it from the builder. - _team_member_max_budget_alert_check( - team_id=_team_id, - team_alias=valid_token.team_alias, - team_metadata=valid_token.team_metadata, - organization_id=valid_token.org_id, - user_id=_user_id, - user_email=user_obj.user_email if user_obj is not None else None, - proxy_logging_obj=proxy_logging_obj, - spend=team_member_spend, - max_budget=team_member_budget, - ) - _entity_id: Final = f"{valid_token.user_id}:{valid_token.team_id}" - raise litellm.BudgetExceededError( - current_cost=team_member_spend, - max_budget=team_member_budget, - message=( - f"Budget has been exceeded! TeamMember={_entity_id} " - f"Current cost: {team_member_spend}, Max budget: {team_member_budget}" - ), - entity_type=Litellm_EntityType.TEAM_MEMBER.value, - entity_id=_entity_id, - ) - - # Check 3. If token is expired - if valid_token.expires is not None: - current_time = datetime.now(timezone.utc) - if isinstance(valid_token.expires, datetime): - expiry_time = valid_token.expires - else: - expiry_time = datetime.fromisoformat(valid_token.expires) - if expiry_time.tzinfo is None or expiry_time.tzinfo.utcoffset(expiry_time) is None: - expiry_time = expiry_time.replace(tzinfo=timezone.utc) - verbose_proxy_logger.debug( - "Checking if token expired, expiry time %s and current time %s", expiry_time, current_time - ) - if expiry_time < current_time: - # Token exists but is expired. - raise ProxyException( - message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", - type=ProxyErrorTypes.expired_key, - code=status.HTTP_401_UNAUTHORIZED, - param=abbreviate_api_key(api_key=api_key), - ) - - if not skip_budget_checks: - with tracer.trace("litellm.proxy.auth.budget_checks"): - # Check 4. Max Budget Alert Check (runs before budget enforcement - # so multi-threshold 100% alerts fire on the request that crosses - # max_budget, before BudgetExceededError is raised below) - await _virtual_key_max_budget_alert_check( - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - user_obj=user_obj, - ) - - # Check 5. Token Spend is under budget - if RouteChecks.is_llm_api_route(route=route): - await _virtual_key_max_budget_check( - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - user_obj=user_obj, - ) - - # Check 6. Soft Budget Check - await _virtual_key_soft_budget_check( - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - user_obj=user_obj, - ) - - # Check 5. Token Model Spend is under Model budget - max_budget_per_model: Final = valid_token.model_max_budget - current_model = _get_model_from_request_context( - request_data=request_data, - route=route, - request=request, - llm_router=llm_router, - team_id=valid_token.team_id, - ) - current_models = _get_model_names_for_budget_checks(model=current_model) - - if ( - max_budget_per_model is not None - and isinstance(max_budget_per_model, dict) - and len(max_budget_per_model) > 0 - and prisma_client is not None - and current_models - and valid_token.token is not None - ): - ## GET THE SPEND FOR THIS MODEL - for model_name in current_models: - await _check_key_model_budget_with_fallback( - valid_token=valid_token, - model_max_budget_limiter=model_max_budget_limiter, - model_name=model_name, - request_data=request_data, - request=request, - llm_model_list=llm_model_list, - llm_router=llm_router, - ) - - # Recompute after a potential budget-fallback rewrite so - # the end-user check below validates the final model - current_model = _get_model_from_request_context( - request_data=request_data, - route=route, - request=request, - llm_router=llm_router, - team_id=valid_token.team_id, - ) - current_models = _get_model_names_for_budget_checks(model=current_model) - - # Check 5a. Internal user model_max_budget - if current_models: - await _check_user_model_budget( - valid_token=valid_token, - model_max_budget_limiter=model_max_budget_limiter, - models=current_models, - ) - - # Check 5b. End-user model max budget - end_user_mmb: Final = valid_token.end_user_model_max_budget - if ( - end_user_mmb is not None - and isinstance(end_user_mmb, dict) - and len(end_user_mmb) > 0 - and current_models - and valid_token.end_user_id is not None - ): - for model_name in current_models: - await model_max_budget_limiter.is_end_user_within_model_budget( - end_user_id=valid_token.end_user_id, - end_user_model_max_budget=end_user_mmb, - model=model_name, - ) - - # Check 6: Additional Common Checks across jwt + key auth - if valid_token.team_id is not None: - try: - if valid_token.team_id == UI_TEAM_ID: - raise TeamNotFoundError(team_id=UI_TEAM_ID) - with tracer.trace("litellm.proxy.auth.get_team_object"): - _team_obj = await get_team_object( - team_id=valid_token.team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) - except HTTPException: - token_team_models: Final = _token_team_models(valid_token) - _team_obj = LiteLLM_TeamTableCachedObj( - team_id=valid_token.team_id, - max_budget=valid_token.team_max_budget, - soft_budget=valid_token.team_soft_budget, - model_max_budget=valid_token.team_model_max_budget, - spend=valid_token.team_spend, - tpm_limit=valid_token.team_tpm_limit, - rpm_limit=valid_token.team_rpm_limit, - tpd_limit=valid_token.team_tpd_limit, - blocked=valid_token.team_blocked, - models=token_team_models, - metadata=valid_token.team_metadata, - object_permission_id=valid_token.team_object_permission_id, - object_permission=await _resolve_object_permission_for_unresolvable_team( - object_permission_id=valid_token.team_object_permission_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ), - ) - else: - _team_obj = None - - if _team_obj is not None: - valid_token.team_object_permission = _team_obj.object_permission - # Keep team_metadata in sync with the freshly fetched team so that - # guardrails (or any other metadata) added after the key was cached - # are picked up on subsequent requests without a cache eviction. - valid_token.team_metadata = _team_obj.metadata - else: - valid_token.team_object_permission = None - - # Fetch project object if key belongs to a project - _project_obj = None - if valid_token.project_id is not None: - _project_obj = await get_project_object( - project_id=valid_token.project_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - if _project_obj is not None: - valid_token.project_metadata = _project_obj.metadata - valid_token.project_alias = _project_obj.project_alias - - global_proxy_spend = None - if litellm.max_budget > 0 and prisma_client is not None: # user set proxy max budget - cache_key: Final = GLOBAL_PROXY_SPEND_CACHE_KEY - with tracer.trace("litellm.proxy.auth.get_global_proxy_spend"): - global_proxy_spend = await _fetch_global_spend_with_event_coordination( - cache_key=cache_key, - user_api_key_cache=user_api_key_cache, - prisma_client=prisma_client, - ) - - if global_proxy_spend is not None: - call_info: Final = CallInfo( - token=valid_token.token, - spend=global_proxy_spend, - max_budget=litellm.max_budget, - user_id=litellm_proxy_admin_name, - team_id=valid_token.team_id, - event_group=Litellm_EntityType.PROXY, - ) - asyncio.create_task( - proxy_logging_obj.budget_alerts( - type="proxy_budget", - user_info=call_info, - ) - ) - # Token passed all checks - if valid_token is None: - raise HTTPException(401, detail="Invalid API key") - if valid_token.token is None: - raise HTTPException(401, detail="Invalid API key, no token associated") - api_key = valid_token.token - - valid_token_dict = valid_token.model_dump(exclude_none=True) - valid_token_dict.pop("token", None) - # budget_throttle_pct is excluded from model_dump (it must not leak - # into serialized responses), so carry the request-scoped decision - # forward by hand to the auth object the rate limiter receives. - if valid_token.budget_throttle_pct is not None: - valid_token_dict["budget_throttle_pct"] = valid_token.budget_throttle_pct - - if _end_user_object is not None: - valid_token_dict.update(end_user_params) - valid_token_dict["end_user_object_permission"] = _end_user_object.object_permission - - # check if token is from litellm-ui, litellm ui makes keys to allow users to login with sso. These keys can only be used for LiteLLM UI functions - # sso/login, ui/login, /key functions and /user functions - # this will never be allowed to call /chat/completions - - if valid_token is None: - # No token was found when looking up in the DB - raise Exception("Invalid proxy server token passed") - if valid_token_dict is not None: - virtual_key_auth_obj: Final = await _return_user_api_key_auth_obj( - user_obj=user_obj, - api_key=api_key, - parent_otel_span=parent_otel_span, - valid_token_dict=valid_token_dict, - route=route, - start_time=start_time, - ) - virtual_key_auth_obj.via_virtual_key = True - return virtual_key_auth_obj + return await validate_resolved_virtual_key( + request=request, + request_data=cast( # cast-ok: model-alias checks must mutate the original request + dict[str, object], request_data + ), + valid_token=valid_token, + api_key=api_key, + route=route, + start_time=start_time, + parent_otel_span=parent_otel_span, + end_user_id=end_user_id, + end_user_params=cast( # cast-ok: builder assembles this dict from validated end-user fields + dict[str, object], end_user_params + ), + _end_user_object=_end_user_object, + ) except Exception as e: return await UserAPIKeyAuthExceptionHandler._handle_authentication_error( e=e, @@ -2583,6 +2208,420 @@ async def _user_api_key_auth_builder( ) +async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of existing shared authorization checks + request: Request, + request_data: dict[str, object], + valid_token: UserAPIKeyAuth | None, + api_key: str, + route: str, + start_time: datetime, + parent_otel_span: Span | None, + end_user_id: str | None, + end_user_params: dict[str, object], + _end_user_object: LiteLLM_EndUserTable | None, +) -> UserAPIKeyAuth: + from litellm.proxy.proxy_server import ( + litellm_proxy_admin_name, + llm_model_list, + llm_router, + model_max_budget_limiter, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if valid_token is not None: + valid_token = _update_key_budget_with_temp_budget_increase(valid_token) + + user_obj: LiteLLM_UserTable | None = None + valid_token_dict: dict = {} + if valid_token is not None: + # Got Valid Token from Cache, DB + # Run checks for + # 1. If token can call model + ## 1a. If token can call fallback models (if client-side fallbacks given) + # 2. If user_id for this token is in budget + # 3. If the user spend within their own team is within budget + # 4. If 'user' passed to /chat/completions, /embeddings endpoint is in budget + # 5. If token is expired + # 6. If token spend is under Budget for the token + # 7. If token spend per model is under budget per model + # 8. If token spend is under team budget + # 9. If team spend is under team budget + + ## base case ## key is disabled + if valid_token.blocked is True: + raise Exception("Key is blocked. Update via `/key/unblock` if you're an admin.") + await _enforce_key_and_fallback_model_access( + valid_token=valid_token, + request_data=request_data, + route=route, + request=request, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + await _prefetch_referenced_auth_objects( + valid_token, end_user_id=end_user_id, user_api_key_cache=user_api_key_cache, prisma_client=prisma_client + ) + + # Check 2. If user_id for this token is in budget - done in common_checks() + if valid_token.user_id is not None: + try: + with tracer.trace("litellm.proxy.auth.get_user_object"): + user_obj = await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as e: + verbose_logger.debug( + "litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - %s", + e, + ) + user_obj = None + + if user_obj is not None: + # The joint verification-token view carries the key's columns only, so the + # user's own per-model budget reaches enforcement and the post-call + # increment through the row fetched here. + valid_token.user_model_max_budget = user_obj.model_max_budget + + if ( + user_obj is not None + and isinstance(user_obj.metadata, dict) + and user_obj.metadata.get("scim_active") is False + ): + raise Exception( + f"User={valid_token.user_id} has been deactivated via SCIM. Keys owned by this user cannot be used." + ) + + # Check 2a. Check if model has zero cost - if so, skip all budget checks + model = _get_model_from_request_context( + request_data=request_data, + route=route, + request=request, + llm_router=llm_router, + team_id=valid_token.team_id, + ) + skip_budget_checks = False + if model is not None and llm_router is not None: + from litellm.proxy.auth.auth_checks import _is_model_cost_zero + + skip_budget_checks = _is_model_cost_zero(model=model, llm_router=llm_router) + if skip_budget_checks: + verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model) + + # Check 3. Check if user is in their team budget + if not skip_budget_checks and valid_token.team_member_spend is not None: + _user_id: Final = valid_token.user_id + _team_id: Final = valid_token.team_id + if prisma_client is not None and _user_id is not None and _team_id is not None: + _cache_key: Final = team_membership_auth_cache_key(team_id=_team_id, user_id=_user_id) + + team_member_info = await user_api_key_cache.async_get_cache( + key=_cache_key, + model_type=LiteLLM_TeamMembership, + ) + if team_member_info is None: + # read from DB + _db_member: Final = await TeamMembershipRepository(prisma_client).table.find_first( + where={ + "user_id": _user_id, + "team_id": _team_id, + }, + include={"litellm_budget_table": True}, + ) + if _db_member is not None: + team_member_info = LiteLLM_TeamMembership(**_db_member.model_dump()) + await user_api_key_cache.async_set_cache( + key=_cache_key, + value=team_member_info, + model_type=LiteLLM_TeamMembership, + ttl=5, + ) + + if team_member_info is not None and team_member_info.litellm_budget_table is not None: + team_member_budget: Final = team_member_info.litellm_budget_table.effective_max_budget( + now=datetime.now(timezone.utc), + ) + if team_member_budget is not None and team_member_budget > 0: + # Read from cross-pod counter (Redis-first) if available + from litellm.proxy.proxy_server import get_current_spend + + team_member_spend = valid_token.team_member_spend + if valid_token.user_id is not None and valid_token.team_id is not None: + team_member_spend = await get_current_spend( + counter_key=f"spend:team_member:{valid_token.user_id}:{valid_token.team_id}", + fallback_spend=team_member_spend, + max_budget=team_member_budget, + ) + if team_member_spend >= team_member_budget: + # common_checks sends this alert on requests that get past here, so only the + # request rejected here sends it from the builder. + _team_member_max_budget_alert_check( + team_id=_team_id, + team_alias=valid_token.team_alias, + team_metadata=valid_token.team_metadata, + organization_id=valid_token.org_id, + user_id=_user_id, + user_email=user_obj.user_email if user_obj is not None else None, + proxy_logging_obj=proxy_logging_obj, + spend=team_member_spend, + max_budget=team_member_budget, + ) + _entity_id: Final = f"{valid_token.user_id}:{valid_token.team_id}" + raise litellm.BudgetExceededError( + current_cost=team_member_spend, + max_budget=team_member_budget, + message=( + f"Budget has been exceeded! TeamMember={_entity_id} " + f"Current cost: {team_member_spend}, Max budget: {team_member_budget}" + ), + entity_type=Litellm_EntityType.TEAM_MEMBER.value, + entity_id=_entity_id, + ) + + # Check 3. If token is expired + if valid_token.expires is not None: + current_time = datetime.now(timezone.utc) + if isinstance(valid_token.expires, datetime): + expiry_time = valid_token.expires + else: + expiry_time = datetime.fromisoformat(valid_token.expires) + if expiry_time.tzinfo is None or expiry_time.tzinfo.utcoffset(expiry_time) is None: + expiry_time = expiry_time.replace(tzinfo=timezone.utc) + verbose_proxy_logger.debug( + "Checking if token expired, expiry time %s and current time %s", expiry_time, current_time + ) + if expiry_time < current_time: + # Token exists but is expired. + raise ProxyException( + message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", + type=ProxyErrorTypes.expired_key, + code=status.HTTP_401_UNAUTHORIZED, + param=abbreviate_api_key(api_key=api_key), + ) + + if not skip_budget_checks: + with tracer.trace("litellm.proxy.auth.budget_checks"): + # Check 4. Max Budget Alert Check (runs before budget enforcement + # so multi-threshold 100% alerts fire on the request that crosses + # max_budget, before BudgetExceededError is raised below) + await _virtual_key_max_budget_alert_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + user_obj=user_obj, + ) + + # Check 5. Token Spend is under budget + if RouteChecks.is_llm_api_route(route=route): + await _virtual_key_max_budget_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + user_obj=user_obj, + ) + + # Check 6. Soft Budget Check + await _virtual_key_soft_budget_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + user_obj=user_obj, + ) + + # Check 5. Token Model Spend is under Model budget + max_budget_per_model: Final = valid_token.model_max_budget + current_model = _get_model_from_request_context( + request_data=request_data, + route=route, + request=request, + llm_router=llm_router, + team_id=valid_token.team_id, + ) + current_models = _get_model_names_for_budget_checks(model=current_model) + + if ( + max_budget_per_model is not None + and isinstance(max_budget_per_model, dict) + and len(max_budget_per_model) > 0 + and prisma_client is not None + and current_models + and valid_token.token is not None + ): + ## GET THE SPEND FOR THIS MODEL + for model_name in current_models: + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=model_max_budget_limiter, + model_name=model_name, + request_data=request_data, + request=request, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + + # Recompute after a potential budget-fallback rewrite so + # the end-user check below validates the final model + current_model = _get_model_from_request_context( + request_data=request_data, + route=route, + request=request, + llm_router=llm_router, + team_id=valid_token.team_id, + ) + current_models = _get_model_names_for_budget_checks(model=current_model) + + # Check 5a. Internal user model_max_budget + if current_models: + await _check_user_model_budget( + valid_token=valid_token, + model_max_budget_limiter=model_max_budget_limiter, + models=current_models, + ) + + # Check 5b. End-user model max budget + end_user_mmb: Final = valid_token.end_user_model_max_budget + if ( + end_user_mmb is not None + and isinstance(end_user_mmb, dict) + and len(end_user_mmb) > 0 + and current_models + and valid_token.end_user_id is not None + ): + for model_name in current_models: + await model_max_budget_limiter.is_end_user_within_model_budget( + end_user_id=valid_token.end_user_id, + end_user_model_max_budget=end_user_mmb, + model=model_name, + ) + + # Check 6: Additional Common Checks across jwt + key auth + if valid_token.team_id is not None: + try: + if valid_token.team_id == UI_TEAM_ID: + raise TeamNotFoundError(team_id=UI_TEAM_ID) + with tracer.trace("litellm.proxy.auth.get_team_object"): + _team_obj = await get_team_object( + team_id=valid_token.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except HTTPException: + token_team_models: Final = _token_team_models(valid_token) + _team_obj = LiteLLM_TeamTableCachedObj( + team_id=valid_token.team_id, + max_budget=valid_token.team_max_budget, + soft_budget=valid_token.team_soft_budget, + model_max_budget=valid_token.team_model_max_budget, + spend=valid_token.team_spend, + tpm_limit=valid_token.team_tpm_limit, + rpm_limit=valid_token.team_rpm_limit, + tpd_limit=valid_token.team_tpd_limit, + blocked=valid_token.team_blocked, + models=token_team_models, + metadata=valid_token.team_metadata, + object_permission_id=valid_token.team_object_permission_id, + object_permission=await _resolve_object_permission_for_unresolvable_team( + object_permission_id=valid_token.team_object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ), + ) + else: + _team_obj = None + + if _team_obj is not None: + valid_token.team_object_permission = _team_obj.object_permission + # Keep team_metadata in sync with the freshly fetched team so that + # guardrails (or any other metadata) added after the key was cached + # are picked up on subsequent requests without a cache eviction. + valid_token.team_metadata = _team_obj.metadata + else: + valid_token.team_object_permission = None + + # Fetch project object if key belongs to a project + _project_obj = None + if valid_token.project_id is not None: + _project_obj = await get_project_object( + project_id=valid_token.project_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if _project_obj is not None: + valid_token.project_metadata = _project_obj.metadata + valid_token.project_alias = _project_obj.project_alias + + global_proxy_spend = None + if litellm.max_budget > 0 and prisma_client is not None: # user set proxy max budget + cache_key: Final = GLOBAL_PROXY_SPEND_CACHE_KEY + with tracer.trace("litellm.proxy.auth.get_global_proxy_spend"): + global_proxy_spend = await _fetch_global_spend_with_event_coordination( + cache_key=cache_key, + user_api_key_cache=user_api_key_cache, + prisma_client=prisma_client, + ) + + if global_proxy_spend is not None: + call_info: Final = CallInfo( + token=valid_token.token, + spend=global_proxy_spend, + max_budget=litellm.max_budget, + user_id=litellm_proxy_admin_name, + team_id=valid_token.team_id, + event_group=Litellm_EntityType.PROXY, + ) + asyncio.create_task( + proxy_logging_obj.budget_alerts( + type="proxy_budget", + user_info=call_info, + ) + ) + # Token passed all checks + if valid_token is None: + raise HTTPException(401, detail="Invalid API key") + if valid_token.token is None: + raise HTTPException(401, detail="Invalid API key, no token associated") + api_key = valid_token.token + + valid_token_dict = valid_token.model_dump(exclude_none=True) + valid_token_dict.pop("token", None) + # budget_throttle_pct is excluded from model_dump (it must not leak + # into serialized responses), so carry the request-scoped decision + # forward by hand to the auth object the rate limiter receives. + if valid_token.budget_throttle_pct is not None: + valid_token_dict["budget_throttle_pct"] = valid_token.budget_throttle_pct + + if _end_user_object is not None: + valid_token_dict.update(end_user_params) + valid_token_dict["end_user_object_permission"] = _end_user_object.object_permission + + # check if token is from litellm-ui, litellm ui makes keys to allow users to login with sso. These keys can only be used for LiteLLM UI functions + # sso/login, ui/login, /key functions and /user functions + # this will never be allowed to call /chat/completions + + if valid_token is None: + # No token was found when looking up in the DB + raise Exception("Invalid proxy server token passed") + if valid_token_dict is not None: + virtual_key_auth_obj: Final = await _return_user_api_key_auth_obj( + user_obj=user_obj, + api_key=api_key, + parent_otel_span=parent_otel_span, + valid_token_dict=valid_token_dict, + route=route, + start_time=start_time, + ) + virtual_key_auth_obj.via_virtual_key = True + return virtual_key_auth_obj + + async def _safe_fetch(label: str, awaitable): """Run an awaitable and return its result. Re-raises authentication / authorization failures (HTTPException, ProxyException, @@ -2718,6 +2757,8 @@ async def _run_centralized_common_checks( request: Request, request_data: dict[str, object], route: str, + *, + force_virtual_key_checks: bool = False, ) -> None: """Run ``common_checks`` once at the ``user_api_key_auth`` wrapper boundary, regardless of which ``_user_api_key_auth_builder`` path @@ -2751,7 +2792,9 @@ async def _run_centralized_common_checks( # auth in the builder — the wrapper must not retroactively apply # authz on top, or k8s readiness probes and other unauthenticated # callers get 401. - if route in LiteLLMRoutes.public_routes.value or route_in_additonal_public_routes(current_route=route): + if not force_virtual_key_checks and ( + route in LiteLLMRoutes.public_routes.value or route_in_additonal_public_routes(current_route=route) + ): return # User-configured pass-through endpoints with ``auth: false`` are @@ -2761,7 +2804,7 @@ async def _run_centralized_common_checks( # admin-only. The "auth" flag on the endpoint config is the # contract; honor it. pass_through_endpoints: Final = general_settings.get("pass_through_endpoints", None) - if pass_through_endpoints is not None: + if not force_virtual_key_checks and pass_through_endpoints is not None: for endpoint in pass_through_endpoints: if isinstance(endpoint, dict) and endpoint.get("path", "") == route and endpoint.get("auth") is not True: return @@ -2772,10 +2815,14 @@ async def _run_centralized_common_checks( # Running common_checks would block every admin route on these # deployments where that was previously not the contract. If any # authn is enabled (JWT, OAuth2, OAuth2-proxy), authz must run. - if is_no_auth_dev_mode(master_key, general_settings): + if not force_virtual_key_checks and is_no_auth_dev_mode(master_key, general_settings): return - if user_custom_auth is not None and not general_settings.get("custom_auth_run_common_checks", False): + if ( + not force_virtual_key_checks + and user_custom_auth is not None + and not general_settings.get("custom_auth_run_common_checks", False) + ): return parent_otel_span: Final = user_api_key_auth_obj.parent_otel_span @@ -3204,6 +3251,8 @@ async def _authorize_authenticated_request( request_data: dict, route: str, api_key: str, + *, + force_virtual_key_checks: bool = False, ) -> UserAPIKeyAuth | None: """Authorize an already-authenticated request: disabled-route check, the single ``common_checks`` gate (which also reserves budget), and end-user fallback @@ -3264,6 +3313,7 @@ async def _authorize_authenticated_request( request=request, request_data=authorized_data, route=route, + force_virtual_key_checks=force_virtual_key_checks, ) except Exception as e: return await UserAPIKeyAuthExceptionHandler._handle_authentication_error( @@ -3883,3 +3933,45 @@ async def _run_post_custom_auth_checks( valid_token.project_alias = _project_obj.project_alias return valid_token + + +async def authorize_internal_virtual_key( + key_hash: str, request: Request, request_data: dict[str, object] +) -> UserAPIKeyAuth: + """Authorize a server-owned job against its persisted virtual-key assignment, never a client-supplied bearer hash.""" + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + identity: Final = IdentityStore.key_from_principal( + await IdentityStore(prisma_client, user_api_key_cache, proxy_logging_obj=proxy_logging_obj).resolve( + hashed_token=key_hash + ) + ) + route: Final = get_request_route(request=request) + await pre_db_read_auth_checks(request_data=request_data, request=request, route=route) + auth: Final = await validate_resolved_virtual_key( + request=request, + request_data=request_data, + valid_token=identity, + api_key=key_hash, + route=route, + start_time=datetime.now(timezone.utc), + parent_otel_span=None, + end_user_id=None, + end_user_params={}, # mutable-ok: existing end-user validation contract + _end_user_object=None, + ) + auth.budget_reservation = None + recovered: Final = await _authorize_authenticated_request( + user_api_key_auth_obj=auth, + request=request, + request_data=request_data, + route=route, + api_key=key_hash, + force_virtual_key_checks=True, + ) + if recovered is not None: + return recovered + _seed_request_destinations(auth, request) + auth.request_route = route + request.state.principal = _resolve_request_principal(request, auth) + return auth diff --git a/litellm/proxy/engine/billing.py b/litellm/proxy/engine/billing.py new file mode 100644 index 00000000000..ca625ed0de6 --- /dev/null +++ b/litellm/proxy/engine/billing.py @@ -0,0 +1,98 @@ +from collections.abc import Awaitable, Callable, Mapping +from typing import Final + +import orjson +from fastapi import HTTPException, Request, Response +from pydantic import TypeAdapter +from starlette.types import Message + +import litellm +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.auth.resolvers.store import IdentityStore +from litellm.proxy.auth.user_api_key_auth import authorize_internal_virtual_key +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.spend_tracking.budget_reservation import release_unbound_budget_reservation +from litellm.types.utils import ModelResponse + + +async def validate_key(key_id: str | None) -> UserAPIKeyAuth | None: + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if key_id is None: + return None + key: Final = IdentityStore.key_from_principal( + await IdentityStore(prisma_client, user_api_key_cache, proxy_logging_obj=proxy_logging_obj).resolve( + hashed_token=key_id + ) + ) + if key.blocked or key.is_session_token: + raise HTTPException(400, "Choose an active virtual key for Lens analysis") + return key + + +async def complete( + key_id: str, data: dict[str, object], reserve: Callable[[], Awaitable[None]], incoming: Request +) -> tuple[ModelResponse, float | None]: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import llm_router, proxy_config, proxy_logging_obj, version + + payload: Final = orjson.dumps(data) + client_ip: Final = IPAddressUtils.get_mcp_client_ip(incoming) + + body: Final[Message] = { + "type": "http.request", + "body": payload, + "more_body": False, + } + messages: Final = iter((body,)) + + async def receive() -> Message: + message: Final = next(messages, None) + return message if message is not None else await incoming.receive() + + request: Final = Request( + { # mutable-ok: Starlette mutates its ASGI scope + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "raw_path": b"/v1/chat/completions", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], # mutable-ok: ASGI header contract + "scheme": incoming.url.scheme or "http", + "client": (client_ip, incoming.client.port if incoming.client else 0) if client_ip else None, + "server": ("litellm.internal", 80), + }, + receive=receive, + ) + try: + auth: Final = await authorize_internal_virtual_key(key_id, request, data) + await reserve() + processor: Final = ProxyBaseLLMRequestProcessing(data=data) + fastapi_response: Final = Response() + try: + response: Final = TypeAdapter(ModelResponse).validate_python( + await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=auth, + route_type="acompletion", + proxy_logging_obj=proxy_logging_obj, + general_settings=TypeAdapter(dict[str, object]).validate_python(proxy_server.general_settings), # pyright: ignore[reportUnknownMemberType] # Validate the legacy untyped config at the request boundary + proxy_config=proxy_config, + llm_router=llm_router, + version=version, + ) + ) + billed: Final = fastapi_response.headers.get("x-litellm-response-cost") + return response, float(billed) if billed not in (None, "", "None") else None + except Exception as exc: + raise await processor._handle_llm_api_exception( # pyright: ignore[reportPrivateUsage] # Standard proxy endpoint failure hook releases limits and records failures + e=exc, user_api_key_dict=auth, proxy_logging_obj=proxy_logging_obj, version=version + ) + except litellm.BudgetExceededError: + raise HTTPException(402, "The analysis key or its owner has reached a budget limit") + finally: + reservation: Final = getattr(request.state, "budget_reservation", None) + if isinstance(reservation, Mapping): + await release_unbound_budget_reservation(TypeAdapter(dict[str, object]).validate_python(reservation)) diff --git a/litellm/proxy/engine/endpoints.py b/litellm/proxy/engine/endpoints.py index 385c6b2ca5d..d085fe7b289 100644 --- a/litellm/proxy/engine/endpoints.py +++ b/litellm/proxy/engine/endpoints.py @@ -6,13 +6,14 @@ from types import MappingProxyType from typing import Annotated, Final, TypeAlias from uuid import uuid4 -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from pydantic import AwareDatetime, BaseModel, Field, TypeAdapter from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper +from litellm.proxy.engine.billing import validate_key from litellm.proxy.engine.models import ( Claim, Engine, @@ -277,28 +278,52 @@ async def preview_sample(body: Preview, auth: Auth) -> Sample: ) -class WorkerName(BaseModel): +class WorkerBilling(BaseModel): + analysis_key_id: str = Field(pattern=r"^[a-f0-9]{64}$") + + +class WorkerName(WorkerBilling): name: str = Field(default="Lens worker", min_length=1, max_length=100) @router.post("/workers/register", response_model=WorkerCreated) async def register_worker(body: WorkerName, auth: Auth) -> WorkerCreated: scope: Final = user_scope(auth, write=True) + await validate_key(body.analysis_key_id) token: Final = "lens-" + secrets.token_urlsafe(40) worker: Final = Worker( - id=str(uuid4()), name=body.name, scope=scope, last_seen=datetime(1970, 1, 1, tzinfo=timezone.utc) + id=str(uuid4()), + name=body.name, + scope=scope, + analysis_key_id=body.analysis_key_id, + last_seen=datetime(1970, 1, 1, tzinfo=timezone.utc), ) await repository().save_worker(worker, hashlib.sha256(token.encode()).hexdigest()) return WorkerCreated(worker=worker, token=token) +@router.put("/workers/{worker_id}/billing-key", response_model=Worker) +async def set_worker_billing(worker_id: str, body: WorkerBilling, auth: Auth) -> Worker: + scope: Final = user_scope(auth, write=True) + worker: Final = next((w for w in await repository().workers() if w.id == worker_id), None) + if worker is None or not can_access(scope, worker.scope): + raise HTTPException(404, "Worker not found") + if worker.revoked: + raise HTTPException(409, "Register a new worker instead of updating revoked access") + await validate_key(body.analysis_key_id) + updated: Final = await repository().set_worker_billing(worker.id, body.analysis_key_id) + if updated is None: + raise HTTPException(409, "Worker access was revoked") + return updated + + @router.delete("/workers/{worker_id}") async def revoke_worker(worker_id: str, auth: Auth) -> bool: scope: Final = user_scope(auth, write=True) worker: Final = next((w for w in await repository().workers() if w.id == worker_id), None) if worker is None or not can_access(scope, worker.scope): raise HTTPException(404, "Worker not found") - await repository().save_worker(worker.model_copy(update=MappingProxyType({"revoked": True}))) + await repository().revoke_worker(worker.id) return True @@ -306,6 +331,8 @@ async def revoke_worker(worker_id: str, auth: Auth) -> bool: async def claim(worker: WorkerAuth, protocol_version: int = 1) -> Claim | None: if protocol_version != 2: raise HTTPException(409, "Upgrade the Lens worker using the current Connect worker command") + if worker.analysis_key_id is None: + raise HTTPException(409, "Assign an analysis key to this worker in Lens setup") now: Final = datetime.now(timezone.utc) await repository().heartbeat(worker.id, now.isoformat()) for candidate in await repository().engines(): @@ -398,11 +425,11 @@ async def content( @router.post("/worker/{engine_id}/{job_id}/model", response_model=ModelResult) -async def model(engine_id: str, job_id: str, body: ModelRequest, worker: WorkerAuth) -> ModelResult: +async def model(engine_id: str, job_id: str, body: ModelRequest, worker: WorkerAuth, request: Request) -> ModelResult: from litellm.proxy.engine.inference import analyze engine, job = await assigned(engine_id, job_id, worker) - return await analyze(repository(), engine, job, worker.id, body) + return await analyze(repository(), engine, job, worker, body, request) @router.post("/worker/{engine_id}/{job_id}/result", response_model=Engine) @@ -487,7 +514,9 @@ async def claim_candidate(candidate: Engine, worker: Worker, now: datetime) -> C scheduled: Final = queue_job(e, now, job_id) if e.settings.enabled and e.next_run_at <= now else e return claim_job(scheduled, worker, now) - updated: Final = required(await repository().update(candidate.id, schedule)) + updated: Final = await repository().update(candidate.id, schedule, changed_only=True) + if updated is None: + return None job: Final = current_job(updated) if job and job.worker_id == worker.id and job.status == "running" and job != current_job(candidate): return Claim(engine_id=updated.id, job=job, findings=updated.findings) diff --git a/litellm/proxy/engine/inference.py b/litellm/proxy/engine/inference.py index 36dbe6e6ffd..687f3832a1b 100644 --- a/litellm/proxy/engine/inference.py +++ b/litellm/proxy/engine/inference.py @@ -2,12 +2,14 @@ from datetime import datetime, timezone from types import MappingProxyType from typing import Final -from fastapi import HTTPException +from fastapi import HTTPException, Request from pydantic import BaseModel, ConfigDict, Field import litellm from litellm.integrations.clickhouse.context import lens_analysis -from litellm.proxy.engine.models import Engine, Job, ModelRequest, ModelResult +from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy +from litellm.proxy.engine.billing import complete, validate_key +from litellm.proxy.engine.models import Engine, Job, ModelRequest, ModelResult, Worker from litellm.proxy.engine.repository import EngineRepository from litellm.proxy.engine.state import current_job, renew_budget, replace_job from litellm.types.utils import CostPerToken, ModelResponse @@ -84,14 +86,20 @@ def quote(deployments: tuple[Deployment, ...], prompt: str) -> float: return ((len((prompt + _SYSTEM).encode()) + 1024) * input_rate + 4096 * output_rate) * 2 -async def analyze(repo: EngineRepository, engine: Engine, job: Job, worker_id: str, body: ModelRequest) -> ModelResult: +async def analyze( + repo: EngineRepository, engine: Engine, job: Job, worker: Worker, body: ModelRequest, request: Request +) -> ModelResult: from litellm.proxy.proxy_server import llm_router if llm_router is None: raise HTTPException(503, "No analysis models are configured") + if worker.analysis_key_id is None: + raise HTTPException(409, "Assign an analysis key to this worker in Lens setup") + billing_key: Final = await validate_key(worker.analysis_key_id) + team_id: Final = billing_key.team_id if billing_key else None deployments: Final = tuple( Deployment.model_validate(d) - for d in llm_router.get_model_list(model_name=job.settings.model, team_id=engine.scope.team_id or None) or () + for d in llm_router.get_model_list(model_name=job.settings.model, team_id=team_id) or () ) if not deployments: raise HTTPException(400, "Analysis model is no longer available") @@ -101,7 +109,13 @@ async def analyze(repo: EngineRepository, engine: Engine, job: Job, worker_id: s def reserve(e: Engine) -> Engine: current: Final = renew_budget(e, now) active: Final = current_job(current) - if active is None or active.id != job.id or active.worker_id != worker_id: + if ( + active is None + or active.id != job.id + or active.worker_id != worker.id + or active.lease_until is None + or active.lease_until <= datetime.now(timezone.utc) + ): raise HTTPException(409, "Job was cancelled or reassigned") if current.spent + estimate > current.settings.monthly_budget: raise HTTPException(402, "Monthly lens budget reached; increase it or wait for next month") @@ -109,28 +123,35 @@ async def analyze(repo: EngineRepository, engine: Engine, job: Job, worker_id: s current, active.model_copy(update=MappingProxyType({"cost": active.cost + estimate})) ).model_copy(update=MappingProxyType({"spent": current.spent + estimate})) - if await repo.update(engine.id, reserve) is None: - raise HTTPException(409, "Could not reserve analysis budget") - with lens_analysis(): - response: Final = await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router forwards provider-specific keyword arguments - model=job.settings.model, - messages=[ # mutable-ok: Router requires OpenAI message dictionaries in a list - {"role": "system", "content": _SYSTEM}, # mutable-ok: provider message dictionary - {"role": "user", "content": body.prompt}, # mutable-ok: provider message dictionary - ], - max_tokens=4096, - stream=False, - timeout=120, - num_retries=0, - disable_fallbacks=True, - response_format={"type": "json_object"}, # mutable-ok: provider response-format JSON object - metadata={ # mutable-ok: Router mutates metadata - "tags": ["litellm-engine"], # mutable-ok: logging callbacks require a tag list - "user_api_key_team_id": engine.scope.team_id, - }, - ) + async def reserve_budget() -> None: + if await repo.update(engine.id, reserve) is None: + raise HTTPException(409, "Could not reserve analysis budget") + + data: Final[dict[str, object]] = { # mutable-ok: proxy processing enriches request data + "model": job.settings.model, + "messages": [ # mutable-ok: OpenAI request contract + {"role": "system", "content": _SYSTEM}, # mutable-ok: OpenAI message contract + {"role": "user", "content": body.prompt}, # mutable-ok: OpenAI message contract + ], + "max_tokens": 4096, + "stream": False, + "timeout": 120, + "num_retries": 0, + "disable_fallbacks": True, + "response_format": {"type": "json_object"}, # mutable-ok: provider response-format JSON + "metadata": { # mutable-ok: request processing enriches metadata + "tags": ["litellm-engine"], # mutable-ok: logging callbacks require a list + "lens_id": engine.id, + "lens_run_id": job.id, + "lens_worker_id": worker.id, + "user_api_key_team_id": team_id, + }, + } + + with lens_analysis(), inherit_message_logging_privacy(True): + response, billed_cost = await complete(worker.analysis_key_id, data, reserve_budget, request) parsed: Final = Completion.model_validate_json(response.model_dump_json()) - cost: Final = completion_charge(deployments, response, estimate) + cost: Final = billed_cost if billed_cost is not None else completion_charge(deployments, response, estimate) def settle(e: Engine) -> Engine: charged: Final = next((j for j in e.jobs if j.id == job.id), None) diff --git a/litellm/proxy/engine/models.py b/litellm/proxy/engine/models.py index 01e05e6745e..33e70ff3eca 100644 --- a/litellm/proxy/engine/models.py +++ b/litellm/proxy/engine/models.py @@ -193,6 +193,7 @@ class Engine(Record): class Worker(Record): + analysis_key_id: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$") id: str name: str scope: Scope diff --git a/litellm/proxy/engine/repository.py b/litellm/proxy/engine/repository.py index e6e9a272e8b..7e3c2f27282 100644 --- a/litellm/proxy/engine/repository.py +++ b/litellm/proxy/engine/repository.py @@ -45,20 +45,24 @@ class EngineRepository: ) return engine - async def update(self, engine_id: str, transform: Callable[[Engine], Engine], attempts: int = 8) -> Engine | None: + async def update( + self, engine_id: str, transform: Callable[[Engine], Engine], attempts: int = 8, *, changed_only: bool = False + ) -> Engine | None: for _ in range(attempts): - completed, updated = await self._try_update(engine_id, transform) + completed, updated = await self._try_update(engine_id, transform, changed_only) if completed: return updated return None - async def _try_update(self, engine_id: str, transform: Callable[[Engine], Engine]) -> tuple[bool, Engine | None]: + async def _try_update( + self, engine_id: str, transform: Callable[[Engine], Engine], changed_only: bool + ) -> tuple[bool, Engine | None]: previous: Final = await self.get(engine_id) if previous is None: return True, None candidate: Final = transform(previous) if candidate == previous: - return True, previous + return True, None if changed_only else previous updated: Final = candidate.model_copy(update=MappingProxyType({"version": previous.version + 1})) rows: Final = _ROWS.validate_python( await self.db.query_raw( @@ -135,6 +139,24 @@ class EngineRepository: 'UPDATE "LiteLLM_EngineWorker" SET data=$1::jsonb WHERE id=$2', worker.model_dump_json(), worker.id ) + async def set_worker_billing(self, worker_id: str, key_id: str) -> Worker | None: + rows: Final = _ROWS.validate_python( + await self.db.query_raw( + """UPDATE "LiteLLM_EngineWorker" + SET data=jsonb_set(data, '{analysis_key_id}', to_jsonb($1::text)) + WHERE id=$2 AND COALESCE((data->>'revoked')::boolean, false)=false RETURNING data""", + key_id, + worker_id, + ) + ) + return Worker.model_validate(rows[0].data) if rows else None + + async def revoke_worker(self, worker_id: str) -> None: + await self.db.execute_raw( + """UPDATE "LiteLLM_EngineWorker" SET data=jsonb_set(data, '{revoked}', 'true') WHERE id=$1""", + worker_id, + ) + async def heartbeat(self, worker_id: str, now: str) -> None: await self.db.execute_raw( """UPDATE "LiteLLM_EngineWorker" SET data=jsonb_set(data, '{last_seen}', to_jsonb($1::text)) WHERE id=$2""", diff --git a/litellm/proxy/engine/state.py b/litellm/proxy/engine/state.py index e4f25dc47d5..3ca6e881234 100644 --- a/litellm/proxy/engine/state.py +++ b/litellm/proxy/engine/state.py @@ -91,8 +91,17 @@ def renew_budget(engine: Engine, now: datetime) -> Engine: def merge_finding(engine: Engine, draft: FindingDraft, revision: int, now: datetime) -> Finding: - identity: Final = hashlib.sha256(f"{engine.id}:{draft.check_id}:{draft.title.lower()}".encode()).hexdigest()[:24] - previous: Final = next((f for f in engine.findings if f.id == (draft.existing_finding_id or identity)), None) + legacy_identity: Final = hashlib.sha256(f"{engine.id}:{draft.check_id}:{draft.title.lower()}".encode()).hexdigest()[ + :24 + ] + identity: Final = hashlib.sha256( + f"{engine.id}:{draft.check_id}:{draft.kind}:{draft.title.lower()}".encode() + ).hexdigest()[:24] + identities: Final = (draft.existing_finding_id, identity, legacy_identity) + previous: Final = next( + (f for f in engine.findings if f.id in identities and f.kind == draft.kind and f.check_id == draft.check_id), + None, + ) occurrences: Final = tuple(sorted(frozenset(e.execution_id for e in draft.evidence if e.role == "support"))) if previous is None: return Finding( diff --git a/litellm/proxy/engine/worker.py b/litellm/proxy/engine/worker.py index d7de75ed73c..e71c57ce143 100644 --- a/litellm/proxy/engine/worker.py +++ b/litellm/proxy/engine/worker.py @@ -1,6 +1,7 @@ import asyncio import logging import os +import sqlite3 from collections.abc import Awaitable, Callable from contextlib import suppress from types import MappingProxyType @@ -82,10 +83,12 @@ class EngineWorker: result: Final = await analyze_sample(claim, sample, read, model, progress) saved: Final = await self.client.post(prefix + "/result", json=result.model_dump(mode="json")) saved.raise_for_status() - except (httpx.HTTPError, ValueError) as exc: + except (httpx.HTTPError, ValueError, OSError, sqlite3.Error) as exc: status: Final = exc.response.status_code if isinstance(exc, httpx.HTTPStatusError) else None message: Final = ( - "Monthly budget reached" + "Worker temporary storage failed. Increase its capacity or reduce analysis parallelism." + if isinstance(exc, (OSError, sqlite3.Error)) + else "Monthly budget reached" if status == 402 else "Analysis interrupted. Check worker connectivity, model configuration, and trace storage." ) diff --git a/tests/integration/spend/test_lens_billing.py b/tests/integration/spend/test_lens_billing.py new file mode 100644 index 00000000000..bd9afdfd954 --- /dev/null +++ b/tests/integration/spend/test_lens_billing.py @@ -0,0 +1,203 @@ +import json +from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows, write_rows +from tests.integration._support.process import owned_proxy +from tests.integration.pricing.test_off_peak_pricing import off_peak_window + + +def delete_lens(engine_id: str) -> None: + write_rows('DELETE FROM "LiteLLM_EngineRun" WHERE engine_id=%s', (engine_id,)) + write_rows('DELETE FROM "LiteLLM_Engine" WHERE id=%s', (engine_id,)) + assert read_rows('SELECT id FROM "LiteLLM_Engine" WHERE id=%s', (engine_id,)) == [] + + +@pytest.mark.parametrize("off_peak", (False, True)) +def test_lens_bills_selected_key_and_rechecks_its_permissions(gateway: Gateway, off_peak: bool) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model( + input_cost_per_token=0.000001, + output_cost_per_token=0.000002, + model_info={ + "off_peak_pricing": { + **off_peak_window(-1, 1), + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.000001, + } + } + if off_peak + else None, + ) + key: Final = scenario.key(models=[model], max_budget=1) + key_id: Final = sha256(key.encode()).hexdigest() + worker: Final = gateway.post( + "/engine/workers/register", {"name": "Billing regression", "analysis_key_id": key_id} + ) + worker_id: Final = string_value(object_value(worker["worker"])["id"]) + scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_EngineWorker" WHERE id=%s', (worker_id,)) + engine: Final = gateway.post( + "/engine", + { + "name": "Billing regression", + "model": model, + "enabled": False, + "context": "Answers should be accurate", + "source": "requests", + }, + ) + engine_id: Final = string_value(engine["id"]) + scenario.cleanups.callback(delete_lens, engine_id) + worker_key: Final = string_value(worker["token"]) + unauthorized: Final = gateway.request( + "POST", "/engine/workers/register", {"name": "Denied", "analysis_key_id": key_id}, key=key + ) + assert unauthorized.status_code == 403, unauthorized.text + with ThreadPoolExecutor(max_workers=8) as pool: + claims: Final = tuple( + pool.map( + lambda _: gateway.request("POST", "/engine/worker/claim?protocol_version=2", {}, key=worker_key), + range(8), + ) + ) + assert all(response.status_code == 200 for response in claims) + winners: Final = tuple(response.json() for response in claims if response.json() is not None) + assert len(winners) == 1 + claim: Final = object_value(winners[0]) + assert claim["engine_id"] == engine_id + job_id: Final = string_value(object_value(claim["job"])["id"]) + path: Final = f"/engine/worker/{engine_id}/{job_id}/model" + result: Final = gateway.post(path, {"prompt": "Inspect this run", "purpose": "extract"}, key=worker_key) + expected: Final = (20 * 0.000001 + 20 * 0.000002) * (0.5 if off_peak else 1) + assert result["cost"] == pytest.approx(expected) + rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (key_id,)), + lambda values: len(values) == 1 and values[0]["spend"] == pytest.approx(expected), + seconds=70, + ) + assert rows[0]["spend"] == pytest.approx(expected) + assert gateway.get(f"/engine/{engine_id}")["spent"] == pytest.approx(expected) + raw_hash: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": "Not a bearer credential"}], + }, + key=key_id, + ) + assert raw_hash.status_code == 401, raw_hash.text + gateway.post("/key/update", {"key": key, "max_budget": expected / 2}) + exhausted: Final = gateway.request( + "POST", path, {"prompt": "Must not run", "purpose": "extract"}, key=worker_key + ) + assert exhausted.status_code == 402, exhausted.text + gateway.post("/key/update", {"key": key, "max_budget": 1, "models": ["unavailable-analysis-model"]}) + restricted: Final = gateway.request( + "POST", path, {"prompt": "Must not run", "purpose": "extract"}, key=worker_key + ) + assert restricted.status_code == 403, restricted.text + gateway.post("/key/block", {"key": key}) + blocked: Final = gateway.request("POST", path, {"prompt": "Must not run", "purpose": "extract"}, key=worker_key) + assert blocked.status_code == 400, blocked.text + assert gateway.get(f"/engine/{engine_id}")["spent"] == pytest.approx(expected) + replacement: Final = scenario.key(models=[model], rpm_limit=1) + replacement_id: Final = sha256(replacement.encode()).hexdigest() + changed: Final = gateway.request( + "PUT", f"/engine/workers/{worker_id}/billing-key", {"analysis_key_id": replacement_id} + ) + assert changed.status_code == 200, changed.text + billed_replacement: Final = gateway.post( + path, {"prompt": "Inspect another run", "purpose": "extract"}, key=worker_key + ) + assert billed_replacement["cost"] == pytest.approx(expected) + limited: Final = gateway.request("POST", path, {"prompt": "Must not run", "purpose": "extract"}, key=worker_key) + assert limited.status_code == 429, limited.text + second_rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (replacement_id,)), + lambda values: len(values) == 1 and values[0]["spend"] == pytest.approx(expected), + seconds=70, + ) + assert second_rows[0]["spend"] == pytest.approx(expected) + revoked: Final = gateway.request("DELETE", f"/engine/workers/{worker_id}") + assert revoked.status_code == 200, revoked.text + denied_worker: Final = gateway.request( + "POST", path, {"prompt": "Must not run", "purpose": "extract"}, key=worker_key + ) + assert denied_worker.status_code == 401, denied_worker.text + forbidden_change: Final = gateway.request( + "PUT", f"/engine/workers/{worker_id}/billing-key", {"analysis_key_id": replacement_id} + ) + assert forbidden_change.status_code == 409, forbidden_change.text + gateway.post(f"/engine/{engine_id}/cancel", {}) + + +@pytest.mark.parametrize("cancel_on_disconnect", (False, True)) +def test_worker_spend_logs_do_not_expose_investigation_content( + gateway: Gateway, tmp_path: Path, cancel_on_disconnect: bool +) -> None: + config: Final = tmp_path / "lens-privacy.json" + config.write_text( + json.dumps( + { + "model_list": [], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "store_model_in_db": True, + "store_prompts_in_spend_logs": True, + "cancel_on_disconnect": cancel_on_disconnect, + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + "allowed_ips": ["127.0.0.1"], + }, + } + ) + ) + with owned_proxy(gateway, tmp_path, {}, config=config) as isolated, isolated.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.000001, output_cost_per_token=0.000002) + key: Final = scenario.key(models=[model]) + key_id: Final = sha256(key.encode()).hexdigest() + marker: Final = "PRIVATE_OTHER_TEAM_TRACE_CONTENT" + ordinary: Final = isolated.chat(model, key=key, text=marker) + retained: Final = eventually( + lambda: read_rows( + 'SELECT proxy_server_request FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (string_value(ordinary["id"]),), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert marker in str(retained[0]), "Control must prove this proxy retains ordinary prompts" + worker: Final = isolated.post("/engine/workers/register", {"analysis_key_id": key_id}) + worker_id: Final = string_value(object_value(worker["worker"])["id"]) + scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_EngineWorker" WHERE id=%s', (worker_id,)) + engine: Final = isolated.post( + "/engine", {"name": "Log privacy", "model": model, "enabled": False, "context": "Find problems"} + ) + engine_id: Final = string_value(engine["id"]) + scenario.cleanups.callback(delete_lens, engine_id) + worker_token: Final = string_value(worker["token"]) + claim: Final = isolated.post("/engine/worker/claim?protocol_version=2", {}, key=worker_token) + job_id: Final = string_value(object_value(claim["job"])["id"]) + result: Final = isolated.post( + f"/engine/worker/{engine_id}/{job_id}/model", {"prompt": marker, "purpose": "extract"}, key=worker_token + ) + assert result["content"], "The worker must still receive model output" + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, proxy_server_request, response FROM "LiteLLM_SpendLogs" WHERE api_key=%s AND request_id<>%s', + (key_id, string_value(ordinary["id"])), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert float(rows[0]["spend"]) == pytest.approx(result["cost"]) + assert marker not in str(rows[0]) + assert result["content"] not in str(rows[0]["response"]) + isolated.post(f"/engine/{engine_id}/cancel", {}) diff --git a/tests/proxy_behavior/lens/test_lifecycle.py b/tests/proxy_behavior/lens/test_lifecycle.py index c8f24bf8e1b..8a9d3873a29 100644 --- a/tests/proxy_behavior/lens/test_lifecycle.py +++ b/tests/proxy_behavior/lens/test_lifecycle.py @@ -1,12 +1,14 @@ +import asyncio import hashlib import os from collections.abc import AsyncIterator from datetime import datetime, timedelta, timezone from typing import Final +from uuid import uuid4 import pytest import pytest_asyncio -from fastapi import HTTPException +from fastapi import HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials from litellm import Router @@ -22,6 +24,14 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging async def lens_database() -> AsyncIterator[PrismaClient]: original_db: Final = proxy_server.prisma_client original_router: Final = proxy_server.llm_router + original_settings: Final = proxy_server.general_settings + proxy_server.general_settings = { + **original_settings, + "allowed_ips": ["127.0.0.1"], + "use_x_forwarded_for": True, + "mcp_trusted_proxy_ranges": ["192.0.2.100/32"], + "mcp_xff_num_trusted_hops": 1, + } client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache())) await client.connect() proxy_server.prisma_client = client @@ -42,6 +52,7 @@ async def lens_database() -> AsyncIterator[PrismaClient]: try: yield client finally: + proxy_server.general_settings = original_settings proxy_server.prisma_client = original_db proxy_server.llm_router = original_router await client.disconnect() @@ -57,7 +68,11 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: checks=(Check(id="retries", instruction="Find unrecovered retries"),), ) engine: Final = await endpoints.create_engine(settings, admin) - registration: Final = await endpoints.register_worker(endpoints.WorkerName(name="Test analyzer"), admin) + key_id: Final = hashlib.sha256(uuid4().bytes).hexdigest() + await lens_database.db.litellm_verificationtoken.create(data={"token": key_id, "models": ["lens-test-analysis"]}) + registration: Final = await endpoints.register_worker( + endpoints.WorkerName(name="Test analyzer", analysis_key_id=key_id), admin + ) credentials: Final = HTTPAuthorizationCredentials(scheme="Bearer", credentials=registration.token) worker: Final = await endpoints.worker_auth(credentials) try: @@ -70,8 +85,12 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: listing: Final = await endpoints.list_engines(admin) assert engine.id in tuple(e.id for e in listing.engines) assert worker.id in tuple(w.id for w in listing.workers) - claimed: Final = await endpoints.claim_candidate(engine, worker, datetime.now(timezone.utc)) - assert claimed is not None + claims: Final = await asyncio.gather( + *(endpoints.claim_candidate(engine, worker, datetime.now(timezone.utc)) for _ in range(8)) + ) + winners: Final = tuple(claim for claim in claims if claim is not None) + assert len(winners) == 1 + claimed: Final = winners[0] assert claimed.job.worker_id == worker.id assert ( await endpoints.claim_candidate( @@ -88,13 +107,80 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: claimed.job.id, ModelRequest(prompt="Return an empty observations list", purpose="extract"), worker, + Request( + { + "type": "http", + "scheme": "http", + "path": "/engine/worker/model", + "headers": [], + "client": ("127.0.0.1", 1234), + } + ), ) assert '"observations"' in response.content + with pytest.raises(HTTPException) as denied_ip: + await endpoints.model( + engine.id, + claimed.job.id, + ModelRequest(prompt="Must not run", purpose="extract"), + worker, + Request( + { + "type": "http", + "scheme": "http", + "path": "/engine/worker/model", + "headers": [(b"x-forwarded-for", b"127.0.0.1")], + "client": ("192.0.2.1", 1234), + } + ), + ) + assert denied_ip.value.status_code == 403 + forwarded: Final = await endpoints.model( + engine.id, + claimed.job.id, + ModelRequest(prompt="Return an empty observations list", purpose="extract"), + worker, + Request( + { + "type": "http", + "scheme": "http", + "path": "/engine/worker/model", + "headers": [(b"x-forwarded-for", b"127.0.0.1")], + "client": ("192.0.2.100", 1234), + } + ), + ) + assert '"observations"' in forwarded.content + with pytest.raises(HTTPException) as spoofed_chain: + await endpoints.model( + engine.id, + claimed.job.id, + ModelRequest(prompt="Must not run", purpose="extract"), + worker, + Request( + { + "type": "http", + "scheme": "http", + "path": "/engine/worker/model", + "headers": [(b"x-forwarded-for", b"127.0.0.1, 192.0.2.1")], + "client": ("192.0.2.100", 1234), + } + ), + ) + assert spoofed_chain.value.status_code == 403 charged: Final = await endpoints.get_engine(engine.id, worker.scope) - assert charged.spent == pytest.approx(response.cost) - assert charged.jobs[0].cost == pytest.approx(response.cost) + assert charged.spent == pytest.approx(response.cost + forwarded.cost) + assert charged.jobs[0].cost == pytest.approx(response.cost + forwarded.cost) + legacy: Final = worker.model_copy(update={"analysis_key_id": None}) + await endpoints.repository().save_worker(legacy) + authenticated_legacy: Final = await endpoints.worker_auth(credentials) + assert authenticated_legacy.analysis_key_id is None + with pytest.raises(HTTPException) as needs_billing: + await endpoints.claim(authenticated_legacy, protocol_version=2) + assert needs_billing.value.status_code == 409 + assert await endpoints.heartbeat(engine.id, claimed.job.id, authenticated_legacy) finished: Final = await endpoints.result( - engine.id, claimed.job.id, Result(coverage=Coverage(screened=2)), worker + engine.id, claimed.job.id, Result(coverage=Coverage(screened=2)), authenticated_legacy ) assert finished.jobs[0].status == "completed" assert finished.jobs[0].coverage.screened == 2 @@ -124,6 +210,10 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: assert cancelled.jobs[0].status == "cancelled" assert await endpoints.cancel_engine(engine.id, admin) == cancelled assert await endpoints.revoke_worker(worker.id, admin) + assert await endpoints.repository().set_worker_billing(worker.id, key_id) is None + with pytest.raises(HTTPException) as revoked_billing: + await endpoints.set_worker_billing(worker.id, endpoints.WorkerBilling(analysis_key_id=key_id), admin) + assert revoked_billing.value.status_code == 409 with pytest.raises(HTTPException) as revoked: await endpoints.worker_auth(credentials) assert revoked.value.status_code == 401 @@ -134,3 +224,4 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: await lens_database.db.execute_raw('DELETE FROM "LiteLLM_EngineRun" WHERE engine_id=$1', engine.id) await lens_database.db.execute_raw('DELETE FROM "LiteLLM_Engine" WHERE id=$1', engine.id) await lens_database.db.execute_raw('DELETE FROM "LiteLLM_EngineWorker" WHERE id=$1', worker.id) + await lens_database.db.execute_raw('DELETE FROM "LiteLLM_VerificationToken" WHERE token=$1', key_id) diff --git a/tests/proxy_behavior/lens/worker_storage_smoke.py b/tests/proxy_behavior/lens/worker_storage_smoke.py new file mode 100644 index 00000000000..8c80915f978 --- /dev/null +++ b/tests/proxy_behavior/lens/worker_storage_smoke.py @@ -0,0 +1,88 @@ +import asyncio +import logging +from datetime import datetime, timezone +from pathlib import Path +from queue import SimpleQueue +from typing import Final + +import httpx +from engine.models import ( + Claim, + EngineSettings, + Execution, + ExecutionContent, + Job, + ModelResult, + Result, + Sample, + TracePart, +) +from engine.worker import EngineWorker + + +async def main() -> None: + now: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + claims: Final = iter(("full", "healthy")) + saved: Final = SimpleQueue[Result]() + pages: Final = SimpleQueue[str]() + settings: Final = EngineSettings(name="Storage recovery", model="unused", context="Finish the task", concurrency=1) + execution: Final = Execution( + id="run", source="traces", trace_id="trace", team_id="", name="Task", start_time="", span_count=10000 + ) + + def handle(request: httpx.Request) -> httpx.Response: + path: Final = request.url.path + if path.endswith("/claim"): + claim: Final = Claim( + engine_id="lens", + job=Job(id=next(claims), created_at=now, start=now, end=now, settings=settings, revision=1), + findings=(), + ) + return httpx.Response(200, json=claim.model_dump(mode="json")) + if path.endswith("/sample"): + return httpx.Response(200, json=Sample(executions=(execution,), eligible=1).model_dump()) + if path.endswith("/content"): + healthy: Final = "/healthy/" in path + cursor: Final = request.url.params.get("cursor", "") + pages.put(cursor) + assert pages.qsize() < 100, "The deliberately small temporary mount must fill" + content: Final = ExecutionContent( + execution=execution, + parts=tuple( + TracePart( + execution_id="run", + span_id=f"{cursor}-{i}", + name="tool", + kind="tool", + content="Finished" if healthy else "x" * 8000, + ) + for i in range(1 if healthy else 40) + ), + next_cursor=None if healthy else str(pages.qsize()), + ) + return httpx.Response(200, json=content.model_dump()) + if path.endswith("/model"): + assert "/healthy/" in path, "Storage failure must occur before spending on analysis" + return httpx.Response(200, json=ModelResult(content='{"observations":[]}', cost=0).model_dump()) + if path.endswith("/result"): + saved.put(Result.model_validate_json(request.content)) + return httpx.Response(200, json=True) + assert path.endswith(("/progress", "/heartbeat")), path + return httpx.Response(200, json=True) + + async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client: + worker: Final = EngineWorker(client) + assert await worker.run_once() + failed: Final = saved.get_nowait() + assert failed.error.startswith("Worker temporary storage failed.") + assert not failed.findings + assert not tuple(Path("/tmp").glob("lens-trace-*")), "Failed scan left temporary files behind" + assert await worker.run_once() + recovered: Final = saved.get_nowait() + assert recovered.error == "" and recovered.coverage.screened == 1 + assert not tuple(Path("/tmp").glob("lens-trace-*")) + logging.info("Storage-full scan failed clearly; temporary files cleaned; next scan completed") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 9cdac341b1f..08c2f02a83c 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -1440,7 +1440,11 @@ def test_jwt_path_enforces_the_user_model_budget_before_returning(): from litellm.proxy.auth import user_api_key_auth as auth_module - tree = ast.parse(textwrap.dedent(inspect.getsource(auth_module._user_api_key_auth_builder))) + tree = ast.parse( + textwrap.dedent(inspect.getsource(auth_module._user_api_key_auth_builder)) + + "\n" + + textwrap.dedent(inspect.getsource(auth_module.validate_resolved_virtual_key)) + ) def calls_before_each_return(node): seen_check = [] @@ -1481,7 +1485,11 @@ def test_every_jwt_branch_carries_the_user_model_budget(): from litellm.proxy.auth import user_api_key_auth as auth_module - tree = ast.parse(textwrap.dedent(inspect.getsource(auth_module._user_api_key_auth_builder))) + tree = ast.parse( + textwrap.dedent(inspect.getsource(auth_module._user_api_key_auth_builder)) + + "\n" + + textwrap.dedent(inspect.getsource(auth_module.validate_resolved_virtual_key)) + ) assignments = [ node @@ -1614,7 +1622,11 @@ def test_zero_cost_models_skip_the_user_budget_check_on_every_path(): from litellm.proxy.auth import user_api_key_auth as auth_module - tree = ast.parse(textwrap.dedent(inspect.getsource(auth_module._user_api_key_auth_builder))) + tree = ast.parse( + textwrap.dedent(inspect.getsource(auth_module._user_api_key_auth_builder)) + + "\n" + + textwrap.dedent(inspect.getsource(auth_module.validate_resolved_virtual_key)) + ) def guarded_by_skip(node: ast.AST, target: ast.AST) -> bool: for parent in ast.walk(node): @@ -1755,7 +1767,11 @@ def test_mapped_key_jwt_falls_through_to_the_shared_user_budget_attach(): from litellm.proxy.auth import user_api_key_auth as auth_module - tree = ast.parse(textwrap.dedent(inspect.getsource(auth_module._user_api_key_auth_builder))) + tree = ast.parse( + textwrap.dedent(inspect.getsource(auth_module._user_api_key_auth_builder)) + + "\n" + + textwrap.dedent(inspect.getsource(auth_module.validate_resolved_virtual_key)) + ) # Half one: the shared block copies the user row's budget onto the token. copies_user_row = [ diff --git a/tests/unit/proxy/engine/test_state.py b/tests/unit/proxy/engine/test_state.py index d731b89964c..8d56f4595da 100644 --- a/tests/unit/proxy/engine/test_state.py +++ b/tests/unit/proxy/engine/test_state.py @@ -200,3 +200,45 @@ def test_batch_snapshot_keeps_feedback_identity_and_only_current_evidence() -> N assert snapshot.title == "Updated wording" assert snapshot.evidence == draft.evidence assert snapshot.revision == 2 + + +@pytest.mark.parametrize("explicit_reference", (False, True)) +def test_issue_and_pattern_with_same_title_keep_independent_feedback(explicit_reference: bool) -> None: + from litellm.proxy.engine.state import snapshot_finding + + original: Final = engine() + issue: Final = merge_finding(original, finding("old"), 1, NOW).model_copy( + update={"status": "dismissed", "reason": "Expected retry"} + ) + reviewed: Final = original.model_copy(update={"findings": (issue,)}) + draft: Final = finding("new").model_copy( + update={"kind": "pattern", "existing_finding_id": issue.id if explicit_reference else None} + ) + pattern: Final = merge_finding(reviewed, draft, 1, NOW) + assert pattern.id != issue.id + assert pattern.kind == "pattern" + assert pattern.status == "open" and pattern.reason == "" + assert pattern.occurrences == ("new",) + assert snapshot_finding(reviewed, draft, 1, NOW).id == pattern.id + both: Final = reviewed.model_copy(update={"findings": (issue, pattern)}) + assert merge_finding(both, finding("again"), 1, NOW).id == issue.id + assert merge_finding(both, finding("again"), 1, NOW).status == "dismissed" + + +def test_legacy_finding_identity_preserves_feedback_only_for_same_kind_and_check() -> None: + import hashlib + + original: Final = engine() + draft: Final = finding("old") + legacy_id: Final = hashlib.sha256(f"{original.id}:{draft.check_id}:{draft.title.lower()}".encode()).hexdigest()[:24] + legacy: Final = merge_finding(original, draft, 1, NOW).model_copy( + update={"id": legacy_id, "status": "dismissed", "reason": "Accepted"} + ) + reviewed: Final = original.model_copy(update={"findings": (legacy,)}) + repeated: Final = merge_finding(reviewed, finding("new"), 2, NOW) + assert repeated.id == legacy_id + assert repeated.status == "dismissed" and repeated.reason == "Accepted" + other: Final = finding("new").model_copy(update={"check_id": "different", "existing_finding_id": legacy_id}) + separate: Final = merge_finding(reviewed, other, 2, NOW) + assert separate.id != legacy_id + assert separate.status == "open" and separate.reason == "" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.integration.test.tsx new file mode 100644 index 00000000000..7a0130bd929 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.integration.test.tsx @@ -0,0 +1,47 @@ +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { apiClient } from "@/components/networking"; +import { AnalysisKey } from "./AnalysisKey"; + +vi.mock("@/components/networking", () => ({ apiClient: { get: vi.fn(), post: vi.fn() } })); + +describe("Lens billing key", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.clearAllMocks(); + }); + it("creates a normal key and only passes its ID to worker settings", async () => { + const user = userEvent.setup(); + const changed = vi.fn(); + vi.mocked(apiClient.get).mockResolvedValue({ keys: [], total_pages: 0 }); + vi.mocked(apiClient.post).mockResolvedValue({ token_id: "b".repeat(64), key: "sk-secret-not-for-settings" }); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: "Create worker key" })); + expect(await screen.findByRole("combobox", { name: "Charge analysis to" })).toHaveValue("Lens: Research"); + expect(apiClient.post).toHaveBeenCalledWith("/key/generate", { + accessToken: "test", + body: { key_alias: "Lens: Research", models: [], metadata: { purpose: "lens" } }, + }); + expect(changed).toHaveBeenCalledExactlyOnceWith("b".repeat(64)); + expect(screen.queryByText("sk-secret-not-for-settings")).not.toBeInTheDocument(); + }); + + it("pages existing keys without dropping the selected billing key", async () => { + const user = userEvent.setup(); + const changed = vi.fn(); + vi.mocked(apiClient.get).mockImplementation(async (_path, options) => ({ + keys: + options?.query?.page === "2" + ? [{ token: "c".repeat(64), key_alias: "Second page" }] + : [{ token: "a".repeat(64), key_alias: "First page" }], + total_pages: 2, + })); + renderWithProviders(); + await user.click(screen.getByRole("combobox", { name: "Charge analysis to" })); + await user.click(await screen.findByRole("option", { name: "Load more keys" })); + await user.click(await screen.findByRole("option", { name: "Second page" })); + expect(changed).toHaveBeenCalledExactlyOnceWith("c".repeat(64)); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.tsx new file mode 100644 index 00000000000..c26c42f5700 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.tsx @@ -0,0 +1,144 @@ +"use client"; + +import { useState } from "react"; +import { useInfiniteQuery } from "@tanstack/react-query"; +import { z } from "zod"; +import { apiClient } from "@/components/networking"; +import { Button } from "@/components/ui/button"; +import { + Combobox, + ComboboxContent, + ComboboxEmpty, + ComboboxInput, + ComboboxItem, + ComboboxList, +} from "@/components/ui/combobox"; + +const keySchema = z.object({ token: z.string(), key_alias: z.string().nullable().optional() }); +const pageSchema = z.object({ keys: z.array(keySchema), total_pages: z.number() }); +type Key = z.infer; + +export function AnalysisKey({ + accessToken, + value, + onChange, + name, +}: { + accessToken: string; + value: string | null; + onChange: (key: string | null) => void; + name: string; +}) { + const [query, setQuery] = useState(""); + const [selected, setSelected] = useState(value ? { token: value } : null); + const [creating, setCreating] = useState(false); + const [error, setError] = useState(""); + const queryOptions = { + queryKey: ["lens-analysis-keys", accessToken, query], + initialPageParam: 1, + queryFn: async ({ pageParam, signal }: { pageParam: number; signal: AbortSignal }) => + pageSchema.parse( + await apiClient.get("/key/list", { + accessToken, + signal, + query: { + page: String(pageParam), + size: "25", + return_full_object: "true", + key_alias: query || undefined, + substring_matching: "true", + include_team_keys: "true", + include_created_by_keys: "true", + status: "active", + }, + }), + ), + getNextPageParam: (lastPage: z.infer, pages: z.infer[]) => + pages.length < lastPage.total_pages ? pages.length + 1 : undefined, + }; + const keyPages = useInfiniteQuery(queryOptions); + const keys = keyPages.data?.pages.flatMap((page) => page.keys) ?? []; + const choice = keys.find((key) => key.token === value) ?? selected; + const loading = keyPages.isFetching; + + const create = async () => { + setCreating(true); + setError(""); + try { + const result = await apiClient.post("/key/generate", { + accessToken, + body: { + key_alias: `Lens: ${name}`, + models: [], + metadata: { purpose: "lens" }, + }, + }); + if (!result.token_id) throw new Error("The proxy did not return the new key's ID"); + const key = { token: result.token_id, key_alias: `Lens: ${name}` }; + setSelected(key); + onChange(key.token); + } catch (cause) { + setError(cause instanceof Error ? cause.message : "Could not create a key"); + } finally { + setCreating(false); + } + }; + const changeKey = (key: Key | null, details: { cancel: () => void }) => { + if (key?.token === "load-more") { + details.cancel(); + if (!loading) void keyPages.fetchNextPage(); + return; + } + setSelected(key); + onChange(key?.token ?? null); + }; + const choices = choice && !keys.some((key) => key.token === choice.token) ? [choice, ...keys] : keys; + const items = keyPages.hasNextPage + ? [...choices, { token: "load-more", key_alias: loading ? "Loading…" : "Load more keys" }] + : choices; + return ( +
+

Charge analysis to

+
+
+ key.key_alias || `${key.token.slice(0, 8)}…`} + isItemEqualToValue={(a: Key, b: Key) => a.token === b.token} + onInputValueChange={(text, details) => { + if (details.reason === "input-change" || details.reason === "input-clear") { + setQuery(text); + } + }} + onValueChange={changeKey} + > + + + {loading ? "Loading keys…" : "No matching keys"} + + {(key: Key) => ( + + {key.key_alias || `${key.token.slice(0, 8)}…`} + + )} + + + +
+ +
+

+ Spend appears under this key in API Keys. Its permissions and limits apply. +

+ {(error || keyPages.error) && ( +

+ {error || keyPages.error?.message} +

+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/EngineView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/EngineView.integration.test.tsx index 86499e5adf0..384dd9284db 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/EngineView.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/EngineView.integration.test.tsx @@ -193,7 +193,14 @@ it("runs saved settings immediately without opening setup", async () => { engines: [engine], tracing_enabled: true, workers: [ - { id: "worker", name: "Worker", revoked: false, scope: engine.scope, last_seen: new Date().toISOString() }, + { + id: "worker", + name: "Worker", + revoked: false, + analysis_key_id: "a".repeat(64), + scope: engine.scope, + last_seen: new Date().toISOString(), + }, ], }; if (path === "/engine/lens/runs") return engine.jobs; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/EngineView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/EngineView.tsx index 178a7512821..1ad36c17299 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/EngineView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/EngineView.tsx @@ -95,7 +95,9 @@ export function EngineView({ accessToken, readOnly = false }: { accessToken: str const showEmpty = !query.isLoading && !query.error && engines.length === 0; const engine = engines.find((e) => e.id === selected) ?? engines[0]; const connected = - query.data?.workers?.some((w) => !w.revoked && query.dataUpdatedAt - Date.parse(w.last_seen) < 120000) ?? false; + query.data?.workers?.some( + (w) => !w.revoked && w.analysis_key_id && query.dataUpdatedAt - Date.parse(w.last_seen) < 120000, + ) ?? false; const historyQuery = { queryKey: ["lens-history", engine?.id, historyOffset, accessToken], enabled: !!engine, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.integration.test.tsx index d116283f467..8fdeab444d8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.integration.test.tsx @@ -1,12 +1,12 @@ import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "@/../tests/test-utils"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import { apiClient } from "@/components/networking"; import { WorkerSetup } from "./WorkerSetup"; vi.mock("@/components/networking", () => ({ - apiClient: { post: vi.fn() }, + apiClient: { get: vi.fn(), post: vi.fn(), put: vi.fn() }, proxyBaseUrl: "https://gateway.example/proxy", })); @@ -18,10 +18,19 @@ const created = { last_seen: "1970-01-01T00:00:00Z", scope: { all_teams: true, api_key_hash: "", team_id: "" }, revoked: false, + analysis_key_id: "b".repeat(64), }, }; describe("Worker setup", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.clearAllMocks(); + vi.mocked(apiClient.get).mockResolvedValue({ + keys: [{ token: "b".repeat(64), key_alias: "Analysis" }], + total_pages: 1, + }); + }); it("generates a complete command using one worker credential and the configured proxy address", async () => { vi.mocked(apiClient.post).mockResolvedValue(created); const user = userEvent.setup(); @@ -29,7 +38,14 @@ describe("Worker setup", () => { expect(screen.getByRole("textbox", { name: "Your LiteLLM deployment URL" })).toHaveValue( "https://gateway.example/proxy", ); + expect(screen.getByRole("button", { name: "Generate setup command" })).toBeDisabled(); + await user.click(screen.getByRole("combobox", { name: "Charge analysis to" })); + await user.click(await screen.findByRole("option", { name: "Analysis" })); await user.click(screen.getByRole("button", { name: "Generate setup command" })); + expect(apiClient.post).toHaveBeenCalledWith("/engine/workers/register", { + accessToken: "admin", + body: { name: "Lens analyzer", analysis_key_id: "b".repeat(64) }, + }); expect(screen.getByRole("status")).toHaveTextContent("Waiting for your analyzer to connect"); await user.click(screen.getByRole("button", { name: "Copy Docker command" })); const command = await navigator.clipboard.readText(); @@ -38,4 +54,28 @@ describe("Worker setup", () => { expect(command).toContain("--add-host host.docker.internal:host-gateway"); expect(command).toContain("ghcr.io/berriai/litellm-lens-worker@sha256:"); }); + it("assigns billing to an existing worker without replacing its access token", async () => { + const user = userEvent.setup(); + const changed = vi.fn(); + vi.mocked(apiClient.put).mockResolvedValue(created.worker); + renderWithProviders( + , + ); + expect(screen.getByText("Billing key required")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Billing key" })); + await user.click(screen.getByRole("combobox", { name: "Charge analysis to" })); + await user.click(await screen.findByRole("option", { name: "Analysis" })); + await user.click(screen.getByRole("button", { name: "Save billing key" })); + expect(apiClient.put).toHaveBeenCalledWith("/engine/workers/worker/billing-key", { + accessToken: "admin", + body: { analysis_key_id: "b".repeat(64) }, + }); + expect(changed).toHaveBeenCalledOnce(); + expect(apiClient.post).not.toHaveBeenCalled(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx index 3052518ce37..7800f445a17 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx @@ -6,10 +6,11 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } f import { Input } from "@/components/ui/input"; import { serverRootPath } from "@/lib/serverRootPath"; import { apiClient, proxyBaseUrl } from "@/components/networking"; +import { AnalysisKey } from "./AnalysisKey"; import type { EngineList, WorkerCreated } from "./engineData"; export const LENS_WORKER_IMAGE = - "ghcr.io/berriai/litellm-lens-worker@sha256:40fdb82113dd4474cb6e833cf28552487d87c8baf61693a1c3fc2863b7968c6a"; + "ghcr.io/berriai/litellm-lens-worker@sha256:c41e932eaf3e4efbcaf8cc5027c7e93021e5b2823f21cb8785cd107e37b91c9a"; function initialProxyAddress(): string { const url = new URL(proxyBaseUrl || serverRootPath, window.location.origin); @@ -21,6 +22,7 @@ export function workerSetupCommand(address: string, token: string): string { const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'"; return [ "docker run -d --restart unless-stopped --read-only --cap-drop ALL", + " --tmpfs /tmp:rw,noexec,nosuid,size=1g", " --security-opt no-new-privileges --platform linux/amd64 --add-host host.docker.internal:host-gateway", ` -e ${quote("LITELLM_URL=" + address)}`, ` -e ${quote("LENS_WORKER_TOKEN=" + token)}`, @@ -28,6 +30,11 @@ export function workerSetupCommand(address: string, token: string): string { ].join(" \\\n"); } +function workerStatus(worker: EngineList["workers"][number], now: number): string { + if (!worker.analysis_key_id) return "Billing key required"; + return now - Date.parse(worker.last_seen) < 120000 ? "Connected · ready to analyze" : "Not connected"; +} + export function WorkerSetup({ accessToken, workers, @@ -44,19 +51,37 @@ export function WorkerSetup({ const timer = window.setInterval(() => setNow(Date.now()), 15000); return () => window.clearInterval(timer); }, []); + const [analysisKey, setAnalysisKey] = useState(null); + const [editingWorker, setEditingWorker] = useState(null); const [address, setAddress] = useState(initialProxyAddress); const [copied, setCopied] = useState(false); const [created, setCreated] = useState(null); const [error, setError] = useState(""); const [busy, setBusy] = useState(false); + const actionLabel = editingWorker ? "Save billing key" : "Generate setup command"; + const editBilling = (worker: EngineList["workers"][number]) => { + setCreated(null); + setEditingWorker(worker.id); + setAnalysisKey(worker.analysis_key_id ?? null); + }; const createWorker = async () => { setBusy(true); setError(""); try { + if (editingWorker) { + await apiClient.put(`/engine/workers/${editingWorker}/billing-key`, { + accessToken, + body: { analysis_key_id: analysisKey }, + }); + setEditingWorker(null); + setAnalysisKey(null); + onChanged(); + return; + } setCreated( await apiClient.post("/engine/workers/register", { accessToken, - body: { name: "Lens analyzer" }, + body: { name: "Lens analyzer", analysis_key_id: analysisKey }, }), ); onChanged(); @@ -80,13 +105,31 @@ export function WorkerSetup({ Lens reads your agents’ logs and finds issues in the background. Run its analyzer once with Docker. - -

- The analyzer connects to this deployment to read logs and save findings. -

+ {!editingWorker && ( + <> + +

+ The analyzer connects to this deployment to read logs and save findings. +

+ + )} + {editingWorker && ( +

+ Billing for {workers.find((worker) => worker.id === editingWorker)?.name} +

+ )} + {!created && ( + + )} {created ? (

Run this command on your server

@@ -115,8 +158,19 @@ export function WorkerSetup({

) : ( - + )} + {editingWorker && ( + )} {workers @@ -125,10 +179,11 @@ export function WorkerSetup({
{worker.name} - - {now - Date.parse(worker.last_seen) < 120000 ? "Connected · ready to analyze" : "Not connected"} - + {workerStatus(worker, now)} + + {timeControls && (